Updating lightcone-cli to new version of ASTRA

README.md

@@ -44,7 +44,7 @@ drafted with any AI coding assistant.
 ## Capabilities
 
 - **Multiverse analysis** — declare methodological decisions with multiple defensible options; `lc` materializes your analysis across every universe you define
-- **Provenance by construction** — every output is committed to git together with a content-addressed manifest and a re-runnable run record; git-annex carries the bytes, so results travel with the repository
+- **Provenance by construction** — every output is committed to git together with its ASTRA provenance record and a re-runnable run record; git-annex carries the bytes, so results travel with the repository
 - **Locked, isolated execution** — a project's environment is `pyproject.toml` + `uv.lock`; recipes run in it under a sandbox (Landlock on Linux, Seatbelt on macOS) that keeps undeclared files out and stray writes contained
 - **Containers and HPC** — declare `[tool.lightcone.image]` and recipes run in a content-addressed image archived in the repository itself; allocate local or Slurm Dask clusters explicitly with `lc compute`, then pass the cluster name or full ID to `run` or `materialize`
 - **Publication view** — declare a license and `lc materialize` maintains an [RO-Crate](https://www.researchobject.org/ro-crate/) of the project and its provenance, ready to archive or deposit

pyproject.toml

@@ -24,7 +24,7 @@ classifiers = [
 ]
 
 dependencies = [
-    "astra-tools==0.2.18",
+    "astra-tools[evidence]>=0.2.19.dev0",
     "click>=8.0",
     "rich>=13.0",
     "git-annex>=10.2026",
@@ -54,9 +54,24 @@ dev = [
 ]
 
 
+# ASTRA 0.1 is not on PyPI yet: both halves are pinned to the commits this
+# engine is tested against. astra-tools names astra-spec by a path beside its
+# own checkout, which a git source does not have, hence the override.
+[tool.uv]
+override-dependencies = [
+    "astra-spec @ git+https://github.com/LightconeResearch/astra-spec@7de4f21e57a7f23231821a5c3a4b6c560274f36b",
+]
+
+[tool.uv.sources]
+astra-tools = { git = "https://github.com/LightconeResearch/astra-tools", rev = "ecbe9504a6ecb8ed303176b51e836d0e4ecc3d9f" }
+
 [project.urls]
 Repository = "https://github.com/LightconeResearch/lightcone-cli"
 
+# What `astra check` asks about an environment a record names.
+[project.entry-points."astra.environments"]
+lightcone = "lightcone.engine.identity:SCHEME"
+
 [project.scripts]
 lc = "lightcone.cli:main"
 # The git-annex wheel's own entry points, re-declared verbatim: an
@@ -105,7 +120,7 @@ explicit_package_bases = true
 mypy_path = "src"
 
 [[tool.mypy.overrides]]
-module = ["astra.*", "rocrate.*"]
+module = ["rocrate.*"]
 ignore_missing_imports = true
 follow_untyped_imports = true
 

src/lightcone/app/__init__.py

@@ -4,8 +4,8 @@
 Jupyter Development Team): a ``LabServerApp`` serving one page, with its
 token, its page config and Jupyter's Contents API. The page reads the
 project through that API and resolves the spec itself; the app's own routes
-(see ``routes``) answer what that API cannot: an output's recorded run, read
-by lc's manifest reader; every output's state, from the engine's own
+(see ``routes``) answer what that API cannot: a result's recorded run, read
+from its record; every result's state, from the engine's own
 classification; and the cited papers in astra's cache, outside the project.
 It serves no settings or themes: the page has no JupyterLab interface to
 configure. The one other extension it loads is

src/lightcone/app/files.py

@@ -161,7 +161,7 @@ async def get(
             target = resolved_path(root, root / model["path"])
             model["watchPaths"] = [
                 target.relative_to(root).as_posix(),
-                assets.manifest_path(Path(model["path"])).as_posix(),
+                *assets.record_places(model["path"]),
             ]
         return model
 

src/lightcone/app/routes.py

@@ -1,74 +1,69 @@
 """The app's own routes: what the page needs and the Contents API cannot give.
 
-An output's recorded run, read by lc's own manifest reader, so the page
-never parses lc's format and the server never has to open dotfiles to it;
-every output's state, classified by the engine exactly as ``lc status``
-does, so the page holds no copy of that rule; and the cited papers in
-astra's paper cache, which lies outside the
-project (``astra paper add`` fills it, and quotes are verified against it).
-The paper routes are adapted from jupyterlab-lightcone's ``routes.py``
-(BSD-3-Clause, Lightcone Research), without its fetch route: nothing here
-writes the cache. Every route needs the token, like the rest of the server.
+A result's recorded run, read from its ASTRA record, so the page never
+parses that format and the server never has to open dotfiles to it;
+every result's state, classified by the engine exactly as ``lc status``
+does, so the page holds no copy of that rule; and the cited
+papers in astra's cache, which lies outside the project (``astra paper
+add`` fills it, and quotes are verified against it). Nothing here writes
+the cache. Every route needs the token, like the rest of the server.
 """
 
 from __future__ import annotations
 
 import asyncio
+import json
 import os
 from pathlib import Path
 from typing import Any
 
-from astra.papers.cache import CachedPaper, PaperCache
+from astra.evidence import papers
+from astra.records import Record
 from jupyter_core.utils import ensure_async
 from jupyter_server.base.handlers import APIHandler
 from jupyter_server.utils import url_path_join
 from tornado import web
+from tornado.httputil import url_concat
 
-from lightcone.app.files import ProjectFileHandler, open_fd
+from lightcone.app.files import open_fd
 from lightcone.engine import assets, materialize
 from lightcone.engine.project import ProjectError, current_project
 
-_PAPERS = PaperCache().cache_dir
 
+class PapersHandler(APIHandler):
+    """Which of the cited DOIs the cache holds a PDF of, and where to get it."""
 
-def _cached() -> dict[str, CachedPaper]:
-    """Every cached paper by DOI as the SDK normalizes it, unversioned first."""
-    papers: dict[str, CachedPaper] = {}
-    for paper in PaperCache().list_papers():
-        doi = paper.metadata.doi.strip().lower()
-        if doi not in papers or paper.metadata.version is None:
-            papers[doi] = paper
-    return papers
+    @web.authenticated
+    async def get(self) -> None:
+        """Answer the repeated ``doi`` arguments from the cache."""
+        found: dict[str, dict[str, str]] = {}
+        for doi in self.get_arguments("doi"):
+            if await asyncio.to_thread(papers.get, doi, None) is not None:
+                url = url_path_join(self.base_url, "lightcone/api/paper")
+                found[doi] = {"pdfUrl": url_concat(url, {"doi": doi})}
+        self.set_header("Cache-Control", "private, no-store")
+        self.finish({"papers": found})
 
 
-class PapersHandler(APIHandler):
-    """Title, authors and PDF of the cited DOIs the cache holds."""
+class PaperHandler(APIHandler):
+    """The PDF the cache holds for one DOI."""
 
     @web.authenticated
     async def get(self) -> None:
-        """Answer the repeated ``doi`` arguments from one scan of the cache."""
-        cached = await asyncio.to_thread(_cached)
-        papers: dict[str, dict[str, str]] = {}
-        for doi in self.get_arguments("doi"):
-            if (paper := cached.get(doi)) is None:
-                continue
-            pdf = paper.pdf_path.relative_to(_PAPERS).as_posix()
-            metadata = {
-                "title": paper.metadata.title,
-                "authors": ", ".join(paper.metadata.authors or []),
-                "pdfUrl": url_path_join(self.base_url, "lightcone/papers", pdf),
-            }
-            papers[doi] = {key: value for key, value in metadata.items() if value}
+        """Send the paper the ``doi`` argument names, or a 404."""
+        pdf = await asyncio.to_thread(papers.get, self.get_query_argument("doi"), None)
+        if pdf is None:
+            raise web.HTTPError(404)
         self.set_header("Cache-Control", "private, no-store")
-        self.finish({"papers": papers})
+        self.finish(pdf, set_content_type="application/pdf")
 
 
 class RunHandler(APIHandler):
-    """The run lc recorded for one output, or ``null`` when there is none."""
+    """The run recorded for one result, or ``null`` when there is none."""
 
     @web.authenticated
     async def get(self) -> None:
-        """Read the manifest beside the output the ``path`` argument names."""
+        """Read the record of the result the ``path`` argument names."""
         # Whether the path is a file in the project is the contents manager's
         # to say, as it does for /files/: outside the root, hidden, absent or
         # a directory is refused before anything is read.
@@ -77,25 +72,33 @@ async def get(self) -> None:
         if not cm.allow_hidden and await ensure_async(cm.is_hidden(path)):
             raise web.HTTPError(404)
         model = await ensure_async(cm.get(path, type="file", content=False))
-        # The sidecar is hidden, but uses the same confined opener as artifacts.
         root = Path(cm.root_dir).resolve()
-        sidecar = assets.manifest_path(root / model["path"])
-        try:
-            with open_fd(root, sidecar, hidden=True) as fd:
-                with os.fdopen(fd, "r", closefd=False) as file:
-                    manifest = assets.loads(file.read())
-        except (web.HTTPError, UnicodeDecodeError):
-            manifest = None
+
+        def read(name: str) -> str | None:
+            # The record is hidden, but uses the same confined opener as artifacts.
+            try:
+                with open_fd(root, root / name, hidden=True) as fd:
+                    with os.fdopen(fd, "r", closefd=False) as file:
+                        return file.read()
+            except (web.HTTPError, UnicodeDecodeError):
+                return None
+
         run: dict[str, Any] | None = None
-        if manifest is not None:
+        for record_path in assets.record_places(model["path"]):
+            if (text := read(record_path)) is None:
+                continue
+            try:
+                record = Record.from_json(json.loads(text))
+            except (ValueError, KeyError, TypeError):
+                continue
             run = {
-                "finishedAt": manifest.finished_at,
-                "gitRevision": manifest.git_sha,
-                "recipe": manifest.recipe,
-                "environment": manifest.env_version,
-                "cliVersion": manifest.lc_version,
-                "inputVersions": manifest.input_versions,
+                "finishedAt": record.finished or "",
+                "gitRevision": record.commit,
+                "environment": record.environment,
+                "cliVersion": record.runner.version,
+                "inputVersions": {item.entry: item.content_hash for item in record.inputs},
             }
+            break
         self.set_header("Cache-Control", "private, no-store")
         self.finish({"run": run})
 
@@ -121,8 +124,7 @@ async def get(self) -> None:
 
 HANDLERS = [
     ("/lightcone/api/papers", PapersHandler),
+    ("/lightcone/api/paper", PaperHandler),
     ("/lightcone/api/run", RunHandler),
     ("/lightcone/api/status", StatusHandler),
-    # Jupyter's own file handler: ranges for pdf.js, 304s on reopening.
-    ("/lightcone/papers/(.*)", ProjectFileHandler, {"path": str(_PAPERS)}),
 ]

src/lightcone/cli/commands.py

@@ -323,7 +323,7 @@ def materialize(
     Execution requires a cluster name or ID as its first positional argument.
     With --check, all positional arguments are targets and no cluster is used.
 
-    Each output is committed together with its manifest, in a commit that
+    Each output is committed together with its record, in a commit that
     records the command that produced it. The git tree needs to be clean
     before the run can start.
 
@@ -333,8 +333,8 @@ def materialize(
     comes out byte-identical stops there instead of cascading.
 
     An output made under an earlier environment is reported as behind and
-    left alone: it is still what the analysis asks for, and the manifest
-    records the environment and the commit that produced it. Pass
+    left alone: it is still what the analysis asks for, and its record
+    names the environment and the commit that produced it. Pass
     --refresh to remake those too.
     """
     from lightcone.engine import container as engine_container

src/lightcone/engine/__init__.py

@@ -1,9 +1,3 @@
-"""The lightcone execution engine.
-
-Rebuild in progress: the engine is being re-added layer by layer on top of
-the normative design spec. Layer 1 is project scaffolding only —
-:mod:`lightcone.engine.project` (convergence) and
-:mod:`lightcone.engine.templates` (the scaffolded files).
-"""
+"""The lightcone execution engine."""
 
 from __future__ import annotations