From 3bbeca517bf9ebbd73c0672f9a110ddf97098fcf Mon Sep 17 00:00:00 2001 From: Val Redchenko Date: Fri, 9 Jan 2026 11:15:01 +0000 Subject: [PATCH 1/2] refactor: sync docs from docs/ to webui MDX, remove Sphinx directives --- docs/explanations.md | 12 +- docs/explanations/decisions.md | 23 +- docs/explanations/smartem-agent-design.md | 8 +- docs/how-to.md | 37 +- docs/how-to/run-e2e-dev-simulation.md | 2 +- docs/reference.md | 19 +- docs/tutorials.md | 7 +- webui/.gitignore | 5 + webui/package-lock.json | 24 +- webui/scripts/generate-mdx-docs.ts | 166 +++ webui/scripts/prebuild.ts | 5 + .../docs/database-schema-drift-prevention.mdx | 215 ---- webui/src/docs/explanations.mdx | 12 - ...kend-agent-communication-system-design.mdx | 1048 ----------------- webui/src/docs/explanations/decisions.mdx | 19 - .../0001-record-architecture-decisions.mdx | 19 - ...002-switched-to-python-copier-template.mdx | 28 - .../0003-message-queue-message-grouping.mdx | 22 - .../decisions/0004-zocalo-dependency-free.mdx | 30 - ...005-detect-secrets-for-secret-scanning.mdx | 52 - ...-secrets-kubernetes-secrets-management.mdx | 73 -- ...minate-smartem-api-circular-dependency.mdx | 77 -- ...nd-to-agent-communication-architecture.mdx | 89 -- .../0009-commit-generated-route-tree.mdx | 110 -- .../0010-shiki-syntax-highlighting.mdx | 52 - .../0011-remove-python-copier-template.mdx | 55 - .../docs/explanations/epu-data-structures.mdx | 76 -- .../explanations/smartem-agent-design.mdx | 868 -------------- .../src/docs/explanations/technical-notes.mdx | 64 - webui/src/docs/how-to.mdx | 36 - .../configure-environment-variables.mdx | 347 ------ webui/src/docs/how-to/configure-logging.mdx | 183 --- .../how-to/container-user-configuration.mdx | 425 ------- webui/src/docs/how-to/containerization.mdx | 123 -- webui/src/docs/how-to/contribute.mdx | 27 - webui/src/docs/how-to/database-migrations.mdx | 280 ----- webui/src/docs/how-to/deploy-kubernetes.mdx | 77 -- webui/src/docs/how-to/development-tools.mdx | 172 --- webui/src/docs/how-to/generate-docs.mdx | 8 - .../docs/how-to/manage-kubernetes-secrets.mdx | 377 ------ webui/src/docs/how-to/run-agent.mdx | 329 ------ webui/src/docs/how-to/run-backend.mdx | 39 - webui/src/docs/how-to/run-container.mdx | 72 -- .../docs/how-to/run-e2e-dev-simulation.mdx | 828 ------------- webui/src/docs/how-to/sync-github-labels.mdx | 159 --- webui/src/docs/how-to/troubleshoot-cli.mdx | 558 --------- .../src/docs/how-to/use-api-documentation.mdx | 205 ---- .../docs/how-to/use-core-http-api-client.mdx | 256 ---- webui/src/docs/how-to/use-mcp-interface.mdx | 507 -------- webui/src/docs/reference.mdx | 13 - webui/src/docs/reference/cli-reference.mdx | 490 -------- webui/src/docs/terms-and-defs.mdx | 46 - webui/src/docs/tutorials.mdx | 5 - webui/src/docs/tutorials/installation.mdx | 87 -- webui/vite.config.ts | 65 +- 55 files changed, 310 insertions(+), 8621 deletions(-) create mode 100644 webui/scripts/generate-mdx-docs.ts delete mode 100644 webui/src/docs/database-schema-drift-prevention.mdx delete mode 100644 webui/src/docs/explanations.mdx delete mode 100644 webui/src/docs/explanations/backend-agent-communication-system-design.mdx delete mode 100644 webui/src/docs/explanations/decisions.mdx delete mode 100644 webui/src/docs/explanations/decisions/0001-record-architecture-decisions.mdx delete mode 100644 webui/src/docs/explanations/decisions/0002-switched-to-python-copier-template.mdx delete mode 100644 webui/src/docs/explanations/decisions/0003-message-queue-message-grouping.mdx delete mode 100644 webui/src/docs/explanations/decisions/0004-zocalo-dependency-free.mdx delete mode 100644 webui/src/docs/explanations/decisions/0005-detect-secrets-for-secret-scanning.mdx delete mode 100644 webui/src/docs/explanations/decisions/0006-sealed-secrets-kubernetes-secrets-management.mdx delete mode 100644 webui/src/docs/explanations/decisions/0007-eliminate-smartem-api-circular-dependency.mdx delete mode 100644 webui/src/docs/explanations/decisions/0008-backend-to-agent-communication-architecture.mdx delete mode 100644 webui/src/docs/explanations/decisions/0009-commit-generated-route-tree.mdx delete mode 100644 webui/src/docs/explanations/decisions/0010-shiki-syntax-highlighting.mdx delete mode 100644 webui/src/docs/explanations/decisions/0011-remove-python-copier-template.mdx delete mode 100644 webui/src/docs/explanations/epu-data-structures.mdx delete mode 100644 webui/src/docs/explanations/smartem-agent-design.mdx delete mode 100644 webui/src/docs/explanations/technical-notes.mdx delete mode 100644 webui/src/docs/how-to.mdx delete mode 100644 webui/src/docs/how-to/configure-environment-variables.mdx delete mode 100644 webui/src/docs/how-to/configure-logging.mdx delete mode 100644 webui/src/docs/how-to/container-user-configuration.mdx delete mode 100644 webui/src/docs/how-to/containerization.mdx delete mode 100644 webui/src/docs/how-to/contribute.mdx delete mode 100644 webui/src/docs/how-to/database-migrations.mdx delete mode 100644 webui/src/docs/how-to/deploy-kubernetes.mdx delete mode 100644 webui/src/docs/how-to/development-tools.mdx delete mode 100644 webui/src/docs/how-to/generate-docs.mdx delete mode 100644 webui/src/docs/how-to/manage-kubernetes-secrets.mdx delete mode 100644 webui/src/docs/how-to/run-agent.mdx delete mode 100644 webui/src/docs/how-to/run-backend.mdx delete mode 100644 webui/src/docs/how-to/run-container.mdx delete mode 100644 webui/src/docs/how-to/run-e2e-dev-simulation.mdx delete mode 100644 webui/src/docs/how-to/sync-github-labels.mdx delete mode 100644 webui/src/docs/how-to/troubleshoot-cli.mdx delete mode 100644 webui/src/docs/how-to/use-api-documentation.mdx delete mode 100644 webui/src/docs/how-to/use-core-http-api-client.mdx delete mode 100644 webui/src/docs/how-to/use-mcp-interface.mdx delete mode 100644 webui/src/docs/reference.mdx delete mode 100644 webui/src/docs/reference/cli-reference.mdx delete mode 100644 webui/src/docs/terms-and-defs.mdx delete mode 100644 webui/src/docs/tutorials.mdx delete mode 100644 webui/src/docs/tutorials/installation.mdx diff --git a/docs/explanations.md b/docs/explanations.md index 73ab289..50612ad 100644 --- a/docs/explanations.md +++ b/docs/explanations.md @@ -2,9 +2,11 @@ Explanations of how it works and why it works that way. -```{toctree} -:maxdepth: 1 -:glob: +## Architecture & Design +- [SmartEM Agent Design](/docs/explanations/smartem-agent-design) - Agent architecture +- [Backend-Agent Communication](/docs/explanations/backend-agent-communication-system-design) - Communication system design +- [EPU Data Structures](/docs/explanations/epu-data-structures) - EPU data format documentation +- [Technical Notes](/docs/explanations/technical-notes) - Technical implementation notes -explanations/* -``` +## Architecture Decision Records +- [Architecture Decisions](/docs/explanations/decisions) - All ADRs diff --git a/docs/explanations/decisions.md b/docs/explanations/decisions.md index 5debc5e..75b127d 100644 --- a/docs/explanations/decisions.md +++ b/docs/explanations/decisions.md @@ -1,14 +1,19 @@ # Architectural Decision Records -Architectural decisions are made throughout a project's lifetime. As a way of keeping track of these decisions, we -record these decisions in Architecture Decision Records (ADRs) listed below. +Architectural decisions are made throughout a project's lifetime. As a way of keeping track of these decisions, we record these decisions in Architecture Decision Records (ADRs) listed below. -```{toctree} -:glob: true -:maxdepth: 1 +## ADRs -decisions/* -``` +- [ADR-0001: Record Architecture Decisions](/docs/explanations/decisions/0001-record-architecture-decisions) +- [ADR-0002: Switched to Python Copier Template (superseded by ADR-0011)](/docs/explanations/decisions/0002-switched-to-python-copier-template) +- [ADR-0003: Message Queue Message Grouping](/docs/explanations/decisions/0003-message-queue-message-grouping) +- [ADR-0004: Zocalo Dependency-Free](/docs/explanations/decisions/0004-zocalo-dependency-free) +- [ADR-0005: Detect Secrets for Secret Scanning](/docs/explanations/decisions/0005-detect-secrets-for-secret-scanning) +- [ADR-0006: Sealed Secrets for Kubernetes](/docs/explanations/decisions/0006-sealed-secrets-kubernetes-secrets-management) +- [ADR-0007: Eliminate SmartEM API Circular Dependency](/docs/explanations/decisions/0007-eliminate-smartem-api-circular-dependency) +- [ADR-0008: Backend to Agent Communication Architecture](/docs/explanations/decisions/0008-backend-to-agent-communication-architecture) +- [ADR-0009: Commit Generated Route Tree](/docs/explanations/decisions/0009-commit-generated-route-tree) +- [ADR-0010: Use Shiki for Syntax Highlighting](/docs/explanations/decisions/0010-shiki-syntax-highlighting) +- [ADR-0011: Remove Python Copier Template](/docs/explanations/decisions/0011-remove-python-copier-template) -For more information on ADRs see this [blog by Michael -Nygard](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions). +For more information on ADRs see this [blog by Michael Nygard](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions). diff --git a/docs/explanations/smartem-agent-design.md b/docs/explanations/smartem-agent-design.md index 838a2a1..a32fa6a 100644 --- a/docs/explanations/smartem-agent-design.md +++ b/docs/explanations/smartem-agent-design.md @@ -554,7 +554,7 @@ incomplete transfers): **Rationale**: 1. **Entity interdependencies**: Parent entities must exist before children can persist. Concurrent processing risks child processing completing before parent processing, artificially creating orphans. -2. **Throughput sufficiency**: Parsing and persisting a single entity takes <10ms in typical cases. Serial processing +2. **Throughput sufficiency**: Parsing and persisting a single entity takes `<10ms` in typical cases. Serial processing sustains ~100 entities/second, far exceeding typical EPU output rates (10-20 entities/second during active collection). 3. **Simplicity**: Serial processing eliminates concurrency bugs, Data Store locking requirements, and orphan @@ -609,7 +609,7 @@ variables or configuration file for deployment-specific tuning. **Rationale**: 1. **Zero latency**: Orphans resolve immediately when their parent appears, not on next retry cycle. Typical resolution - time: <1ms after parent persistence. + time: `<1ms` after parent persistence. 2. **No wasted cycles**: Resolution attempts only occur when new parents arrive. Periodic retry wastes CPU checking orphans during quiet periods. 3. **Cascading resolution**: Resolving an orphan (e.g., GridSquare) immediately enables resolution of its children @@ -743,7 +743,7 @@ and multiple timing modes: - Zero entity loss across all ordering scenarios and timing modes - Orphan resolution success rate >99% for complete hierarchies - Orphaned entities with missing parents log timeout warnings (no eviction) -- Processing latency <5 seconds for 95th percentile entities (fast mode) +- Processing latency `<5 seconds` for 95th percentile entities (fast mode) #### End-to-End Tests @@ -778,7 +778,7 @@ patterns. Primary test dataset: bi37708-42 (8,389 events). Tests validate: - Memory consumption during processing **Decision Criteria**: Select batch size minimising orphan resolution time whilst maintaining total processing time -<10% above minimum observed (to avoid sacrificing throughput for marginal resolution improvements). +`<10%` above minimum observed (to avoid sacrificing throughput for marginal resolution improvements). #### 2. Persistent Orphan Storage diff --git a/docs/how-to.md b/docs/how-to.md index f9cca90..99ec4d4 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -2,10 +2,35 @@ Practical step-by-step guides for the more experienced user. -```{toctree} -:maxdepth: 1 -:glob: +## Getting Started +- [Run Backend](/docs/how-to/run-backend) - Start the SmartEM backend services +- [Run Agent](/docs/how-to/run-agent) - Run the SmartEM agent +- [Run Container](/docs/how-to/run-container) - Run services in containers -how-to/use-api-documentation -how-to/* -``` +## Development +- [Run E2E Dev Simulation](/docs/how-to/run-e2e-dev-simulation) - End-to-end development testing +- [Development Tools](/docs/how-to/development-tools) - Available development tooling +- [Troubleshoot CLI](/docs/how-to/troubleshoot-cli) - CLI troubleshooting guide +- [Generate Docs](/docs/how-to/generate-docs) - Build documentation locally + +## Configuration +- [Configure Environment Variables](/docs/how-to/configure-environment-variables) - Environment setup +- [Configure Logging](/docs/how-to/configure-logging) - Logging configuration +- [Container User Configuration](/docs/how-to/container-user-configuration) - Container user setup + +## Deployment +- [Containerization](/docs/how-to/containerization) - Build container images +- [Deploy Kubernetes](/docs/how-to/deploy-kubernetes) - Kubernetes deployment guide +- [Manage Kubernetes Secrets](/docs/how-to/manage-kubernetes-secrets) - Secrets management + +## Database +- [Database Migrations](/docs/how-to/database-migrations) - Alembic migration guide + +## APIs & Integration +- [Use API Documentation](/docs/how-to/use-api-documentation) - Interactive API docs +- [Use HTTP API Client](/docs/how-to/use-core-http-api-client) - Core HTTP client usage +- [Use MCP Interface](/docs/how-to/use-mcp-interface) - MCP interface guide + +## Contributing +- [Contribute](/docs/how-to/contribute) - Contribution guidelines +- [Sync GitHub Labels](/docs/how-to/sync-github-labels) - Label synchronization diff --git a/docs/how-to/run-e2e-dev-simulation.md b/docs/how-to/run-e2e-dev-simulation.md index f296a6c..8c82833 100644 --- a/docs/how-to/run-e2e-dev-simulation.md +++ b/docs/how-to/run-e2e-dev-simulation.md @@ -805,7 +805,7 @@ curl -u guest:guest http://localhost:30673 # pragma: allowlist secret - Confirm file permissions allow agent to read files ### Consumer Not Processing Messages -- Verify `RABBITMQ_URL=amqp://guest:guest@localhost:30672/` is correct +- Verify `RABBITMQ_URL=amqp://guest:guest@localhost:30672/` is correct {/* pragma: allowlist secret */} - Check queue has messages waiting - Review consumer logs for connection errors - Ensure database is accessible to worker diff --git a/docs/reference.md b/docs/reference.md index b6e2f16..19a3600 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -2,13 +2,12 @@ Technical reference material including APIs and release notes. -```{toctree} -:maxdepth: 1 -:glob: - -Athena Decision Service API -SmartEM Core API -API <_api/smartem_backend> -genindex -Release Notes -``` +## CLI +- [CLI Reference](/docs/reference/cli-reference) - Command-line interface documentation + +## API Documentation +- [Athena Decision Service API](/api/athena/index.html) - Decision service API docs +- [SmartEM Core API](/api/smartem/index.html) - Core API docs + +## External Links +- [Release Notes](https://github.com/DiamondLightSource/smartem-decisions/releases) - GitHub releases diff --git a/docs/tutorials.md b/docs/tutorials.md index 1fe66c5..17a0052 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -2,9 +2,4 @@ Tutorials for installation and typical usage. New users start here. -```{toctree} -:maxdepth: 1 -:glob: - -tutorials/* -``` +- [Installation](/docs/tutorials/installation) - Get started with SmartEM diff --git a/webui/.gitignore b/webui/.gitignore index 8cd9d60..a45c070 100644 --- a/webui/.gitignore +++ b/webui/.gitignore @@ -26,3 +26,8 @@ lerna-debug.log* # Generated files that should be committed (see ADR 0009) # src/routeTree.gen.ts - committed for immediate type safety # src/config/webui-app-contents.ts - committed for CI + +# Generated documentation (synced from docs/ by scripts/sync-docs.ts) +# These files are regenerated during prebuild, do not edit directly +src/docs/**/*.mdx +!src/docs/index.mdx diff --git a/webui/package-lock.json b/webui/package-lock.json index 2401995..91a315d 100644 --- a/webui/package-lock.json +++ b/webui/package-lock.json @@ -68,7 +68,6 @@ "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.5.tgz", "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.5", @@ -570,7 +569,6 @@ "resolved": "https://registry.npmjs.org/@emotion/react/-/react-11.14.0.tgz", "integrity": "sha512-O000MLDBDdk/EohJPFUqvnp4qnHeYkVP5B0xEG0D/L7cOKP9kefu2DXn8dj74cQfsEzUqh+sr1RzFqiL1o+PpA==", "license": "MIT", - "peer": true, "dependencies": { "@babel/runtime": "^7.18.3", "@emotion/babel-plugin": "^11.13.5", @@ -614,7 +612,6 @@ "resolved": "https://registry.npmjs.org/@emotion/styled/-/styled-11.14.1.tgz", "integrity": "sha512-qEEJt42DuToa3gurlH4Qqc1kVpNq8wO8cJtDzU46TjlzWjDlsVyevtYCRijVq3SrHsROS+gVQ8Fnea108GnKzw==", "license": "MIT", - "peer": true, "dependencies": { "@babel/runtime": "^7.18.3", "@emotion/babel-plugin": "^11.13.5", @@ -1253,7 +1250,6 @@ "resolved": "https://registry.npmjs.org/@mui/material/-/material-7.3.6.tgz", "integrity": "sha512-R4DaYF3dgCQCUAkr4wW1w26GHXcf5rCmBRHVBuuvJvaGLmZdD8EjatP80Nz5JCw0KxORAzwftnHzXVnjR8HnFw==", "license": "MIT", - "peer": true, "dependencies": { "@babel/runtime": "^7.28.4", "@mui/core-downloads-tracker": "^7.3.6", @@ -2231,7 +2227,6 @@ "resolved": "https://registry.npmjs.org/@tanstack/react-router/-/react-router-1.144.0.tgz", "integrity": "sha512-GmRyIGmHtGj3VLTHXepIwXAxTcHyL5W7Vw7O1CnVEtFxQQWKMVOnWgI7tPY6FhlNwMKVb3n0mPFWz9KMYyd2GA==", "license": "MIT", - "peer": true, "dependencies": { "@tanstack/history": "1.141.0", "@tanstack/react-store": "^0.8.0", @@ -2297,7 +2292,6 @@ "resolved": "https://registry.npmjs.org/@tanstack/router-core/-/router-core-1.144.0.tgz", "integrity": "sha512-6oVERtK9XDHCP4XojgHsdHO56ZSj11YaWjF5g/zw39LhyA6Lx+/X86AEIHO4y0BUrMQaJfcjdAQMVSAs6Vjtdg==", "license": "MIT", - "peer": true, "dependencies": { "@tanstack/history": "1.141.0", "@tanstack/store": "^0.8.0", @@ -2646,7 +2640,6 @@ "integrity": "sha512-W609buLVRVmeW693xKfzHeIV6nJGGz98uCPfeXI1ELMLXVeKYZ9m15fAMSaUPBHYLGFsVRcMmSCksQOrZV9BYA==", "devOptional": true, "license": "MIT", - "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -2668,7 +2661,6 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.7.tgz", "integrity": "sha512-MWtvHrGZLFttgeEj28VXHxpmwYbor/ATPYbBfSFZEIRK0ecCFLl2Qo55z52Hss+UV9CRN7trSeq1zbgx7YDWWg==", "license": "MIT", - "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -2729,7 +2721,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -2904,7 +2895,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -3127,8 +3117,7 @@ "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/debug": { "version": "4.4.3", @@ -5811,7 +5800,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.3.tgz", "integrity": "sha512-Ku/hhYbVjOQnXDZFv2+RibmLFGwFdeeKHFcOTlrt7xplBnya5OGn/hIRDsqDiSUcfORsDC7MPxwork8jBwsIWA==", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -5821,7 +5809,6 @@ "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.3.tgz", "integrity": "sha512-yELu4WmLPw5Mr/lmeEpox5rw3RETacE++JgHqQzd2dg+YbJuat3jH4ingc+WPZhxaoFzdv9y33G+F7Nl5O0GBg==", "license": "MIT", - "peer": true, "dependencies": { "scheduler": "^0.27.0" }, @@ -6187,7 +6174,6 @@ "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.53.3.tgz", "integrity": "sha512-w8GmOxZfBmKknvdXU1sdM9NHcoQejwF/4mNgj2JuEEdRaHwwF12K7e9eXn1nLZ07ad+du76mkVsyeb2rKGllsA==", "license": "MIT", - "peer": true, "dependencies": { "@types/estree": "1.0.8" }, @@ -6244,7 +6230,6 @@ "resolved": "https://registry.npmjs.org/seroval/-/seroval-1.4.2.tgz", "integrity": "sha512-N3HEHRCZYn3cQbsC4B5ldj9j+tHdf4JZoYPlcI4rRYu0Xy4qN8MQf1Z08EibzB0WpgRG5BGK08FTrmM66eSzKQ==", "license": "MIT", - "peer": true, "engines": { "node": ">=10" } @@ -6266,7 +6251,6 @@ "resolved": "https://registry.npmjs.org/shiki/-/shiki-3.20.0.tgz", "integrity": "sha512-kgCOlsnyWb+p0WU+01RjkCH+eBVsjL1jOwUYWv0YDWkM2/A46+LDKVs5yZCUXjJG6bj4ndFoAg5iLIIue6dulg==", "license": "MIT", - "peer": true, "dependencies": { "@shikijs/core": "3.20.0", "@shikijs/engine-javascript": "3.20.0", @@ -6283,6 +6267,7 @@ "resolved": "https://registry.npmjs.org/solid-js/-/solid-js-1.9.10.tgz", "integrity": "sha512-Coz956cos/EPDlhs6+jsdTxKuJDPT7B5SVIWgABwROyxjY7Xbr8wkzD68Et+NxnV7DLJ3nJdAC2r9InuV/4Jew==", "license": "MIT", + "peer": true, "dependencies": { "csstype": "^3.1.0", "seroval": "~1.3.0", @@ -6304,6 +6289,7 @@ "resolved": "https://registry.npmjs.org/seroval-plugins/-/seroval-plugins-1.3.3.tgz", "integrity": "sha512-16OL3NnUBw8JG1jBLUoZJsLnQq0n5Ua6aHalhJK4fMQkz1lqR7Osz1sA30trBtd9VUDc2NgkuRCn8+/pBwqZ+w==", "license": "MIT", + "peer": true, "engines": { "node": ">=10" }, @@ -6488,7 +6474,6 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, @@ -6580,7 +6565,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -6809,7 +6793,6 @@ "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.0.tgz", "integrity": "sha512-dZwN5L1VlUBewiP6H9s2+B3e3Jg96D0vzN+Ry73sOefebhYr9f94wwkMNN/9ouoU8pV1BqA1d1zGk8928cx0rg==", "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -6921,7 +6904,6 @@ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "license": "MIT", - "peer": true, "engines": { "node": ">=12" }, diff --git a/webui/scripts/generate-mdx-docs.ts b/webui/scripts/generate-mdx-docs.ts new file mode 100644 index 0000000..55173c1 --- /dev/null +++ b/webui/scripts/generate-mdx-docs.ts @@ -0,0 +1,166 @@ +/** + * Sync documentation from docs/ to webui/src/docs/ + * + * This script copies markdown files from the docs/ directory to webui/src/docs/ + * as MDX files, preserving the directory structure. Generated MDX files are + * gitignored and regenerated during prebuild. + * + * Manual files (index.mdx, navigation.ts) are preserved and not overwritten. + */ + +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs' +import { dirname, join, relative, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +const __dirname = dirname(fileURLToPath(import.meta.url)) +const projectRoot = resolve(__dirname, '..') +const docsSource = resolve(projectRoot, '..', 'docs') +const docsTarget = resolve(projectRoot, 'src', 'docs') + +// Files to exclude from sync (Sphinx-specific or handled separately) +const EXCLUDED_FILES = new Set([ + 'index.md', // Uses React components in webui version + 'genindex.md', // Sphinx-specific + 'athena-decision-service-api-spec.README.md', // API spec file +]) + +// Directories to exclude from sync +const EXCLUDED_DIRS = new Set(['_templates', 'images', 'api', '_api', '_static']) + +// Files that are manually maintained (do not delete or overwrite) +const MANUAL_FILES = new Set(['index.mdx', 'navigation.ts']) + +// Note: We don't add a header comment because JSX-style comments {/* */} +// can interfere with MDX parsing of curly braces in content. +const GENERATED_HEADER = '' + +interface SyncStats { + copied: number + deleted: number + skipped: number +} + +/** + * Recursively collect all .md files from source directory + */ +function collectMdFiles(dir: string, baseDir: string = dir): string[] { + const files: string[] = [] + + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const fullPath = join(dir, entry.name) + + if (entry.isDirectory()) { + if (!EXCLUDED_DIRS.has(entry.name)) { + files.push(...collectMdFiles(fullPath, baseDir)) + } + } else if (entry.isFile() && entry.name.endsWith('.md')) { + if (!EXCLUDED_FILES.has(entry.name)) { + files.push(relative(baseDir, fullPath)) + } + } + } + + return files +} + +/** + * Recursively collect all .mdx files from target directory + */ +function collectMdxFiles(dir: string, baseDir: string = dir): string[] { + const files: string[] = [] + + if (!existsSync(dir)) { + return files + } + + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const fullPath = join(dir, entry.name) + + if (entry.isDirectory()) { + files.push(...collectMdxFiles(fullPath, baseDir)) + } else if (entry.isFile() && entry.name.endsWith('.mdx')) { + if (!MANUAL_FILES.has(entry.name)) { + files.push(relative(baseDir, fullPath)) + } + } + } + + return files +} + +/** + * Sync documentation files from docs/ to webui/src/docs/ + */ +export function syncDocs(): SyncStats { + const stats: SyncStats = { copied: 0, deleted: 0, skipped: 0 } + + console.log('Syncing documentation from docs/ to webui/src/docs/...') + + // Collect source and target files + const sourceMdFiles = collectMdFiles(docsSource) + const targetMdxFiles = collectMdxFiles(docsTarget) + + // Build set of expected target files (source files with .mdx extension) + const expectedTargetFiles = new Set(sourceMdFiles.map((f) => f.replace(/\.md$/, '.mdx'))) + + // Delete stale MDX files (exist in target but not in source) + for (const targetFile of targetMdxFiles) { + if (!expectedTargetFiles.has(targetFile)) { + const fullPath = join(docsTarget, targetFile) + rmSync(fullPath) + console.log(` Deleted: ${targetFile}`) + stats.deleted++ + } + } + + // Copy source files to target + for (const sourceFile of sourceMdFiles) { + const sourcePath = join(docsSource, sourceFile) + const targetFile = sourceFile.replace(/\.md$/, '.mdx') + const targetPath = join(docsTarget, targetFile) + + // Create target directory if needed + const targetDir = dirname(targetPath) + if (!existsSync(targetDir)) { + mkdirSync(targetDir, { recursive: true }) + } + + // Read source content and prepend generated header + const sourceContent = readFileSync(sourcePath, 'utf-8') + const targetContent = GENERATED_HEADER + sourceContent + + // Check if target exists and has same content (skip if unchanged) + if (existsSync(targetPath)) { + const existingContent = readFileSync(targetPath, 'utf-8') + if (existingContent === targetContent) { + stats.skipped++ + continue + } + } + + // Write target file + writeFileSync(targetPath, targetContent) + console.log(` Copied: ${sourceFile} -> ${targetFile}`) + stats.copied++ + } + + console.log( + `Sync complete: ${stats.copied} copied, ${stats.deleted} deleted, ${stats.skipped} unchanged` + ) + + return stats +} + +// Run if executed directly (not imported) +const isMainModule = import.meta.url === `file://${process.argv[1]}` +if (isMainModule) { + syncDocs() +} diff --git a/webui/scripts/prebuild.ts b/webui/scripts/prebuild.ts index 57bad5f..f0aa8c4 100644 --- a/webui/scripts/prebuild.ts +++ b/webui/scripts/prebuild.ts @@ -2,6 +2,11 @@ import { existsSync, readFileSync, writeFileSync } from 'node:fs' import { dirname, resolve } from 'node:path' import { fileURLToPath } from 'node:url' +// Sync documentation from docs/ to webui/src/docs/ +import { syncDocs } from './generate-mdx-docs.js' + +syncDocs() + const __dirname = dirname(fileURLToPath(import.meta.url)) const projectRoot = resolve(__dirname, '..') const coreDir = resolve(projectRoot, '..', 'core') diff --git a/webui/src/docs/database-schema-drift-prevention.mdx b/webui/src/docs/database-schema-drift-prevention.mdx deleted file mode 100644 index f4db9e5..0000000 --- a/webui/src/docs/database-schema-drift-prevention.mdx +++ /dev/null @@ -1,215 +0,0 @@ -# Database Schema Drift Prevention - -This document explains the automated schema drift detection system implemented in SmartEM Decisions to ensure database schema consistency across environments. - -## What is Schema Drift? - -Schema drift occurs when SQLModel definitions (your Python data models) change but the corresponding Alembic database migrations haven't been updated. This creates a mismatch between: - -- What your code expects the database schema to look like (SQLModel definitions) -- What the actual database schema looks like (current migration state) - -This can lead to runtime errors, deployment failures, and data integrity issues. - -## How Our Detection Works - -Our schema drift detection system automatically catches these mismatches in CI/CD by: - -1. **Setting up a temporary database** - Creates a clean PostgreSQL database for testing -2. **Applying existing migrations** - Runs all current Alembic migrations to establish the "official" schema state -3. **Comparing with SQLModel definitions** - Uses Alembic's autogenerate feature to detect what new migrations would be needed -4. **Reporting drift** - Fails the CI build if any schema changes are detected - -## When the Check Runs - -The schema drift check runs automatically: - -- On every pull request to `main` -- On every push to `main` -- On manual workflow dispatch -- Skipped for draft PRs with `[WIP]` prefix - -## What Happens When Drift is Detected - -If schema drift is detected, the CI build will fail with a clear error message: - -``` -SCHEMA DRIFT DETECTED! - -Your SQLModel definitions have changed but migrations haven't been updated. -This means the database schema is out of sync with your model definitions. - -To fix this: -1. Run: alembic revision --autogenerate -m 'description of changes' -2. Review the generated migration file -3. Test the migration: alembic upgrade head -4. Commit the new migration file -``` - -## How to Fix Schema Drift - -When you modify SQLModel definitions (add/remove fields, change types, etc.), follow these steps: - -### 1. Generate the Migration - -```bash -# Activate your virtual environment -source .venv/bin/activate - -# Generate a new migration based on model changes -alembic revision --autogenerate -m "Add user_preferences table" -``` - -### 2. Review the Generated Migration - -Alembic will create a new file in `src/smartem_backend/migrations/versions/`. **Always review this file** to ensure: - -- The changes match your intentions -- No unintended schema modifications are included -- Data migration logic is correct (if needed) - -### 3. Test the Migration - -```bash -# Test applying the migration -alembic upgrade head - -# Test downgrading (optional but recommended) -alembic downgrade -1 -alembic upgrade head -``` - -### 4. Commit the Migration - -```bash -git add src/smartem_backend/migrations/versions/[new_migration_file].py -git commit -m "Add migration for user preferences table" -``` - -## Running the Check Locally - -You can run the schema drift check locally before pushing: - -```bash -# Ensure you have a PostgreSQL instance running and .env.k8s.development configured -python tools/check_schema_drift.py -``` - -The script will: -- Create a temporary database -- Apply existing migrations -- Check for drift -- Clean up the temporary database -- Exit with code 0 (success) or 1 (drift detected) - -## Advanced Usage - -### Custom Database Configuration - -The script uses environment variables from `.env.k8s.development` or similar K8s environment files: - -```bash -POSTGRES_HOST=localhost -POSTGRES_PORT=5432 -POSTGRES_USER=your_user -POSTGRES_PASSWORD=your_password -``` - -### Debugging Migration Issues - -If you encounter issues with the drift detection: - -1. **Check migration files** - Ensure all migration files are committed -2. **Verify database connectivity** - Test connection with your PostgreSQL instance -3. **Review SQLModel imports** - Make sure all models are properly imported in `smartem_backend/model/database.py` -4. **Check Alembic configuration** - Verify `alembic.ini` and `env.py` are correctly configured - -### CI/CD Integration Details - -The schema drift check is implemented as a reusable GitHub Actions workflow: - -- **Workflow file**: `.github/workflows/_schema_drift.yml` -- **Called from**: `.github/workflows/ci.yml` -- **Dependencies**: Runs after basic checks pass -- **Services**: Uses PostgreSQL 15 service container - -## Best Practices - -### For Developers - -1. **Always run locally first** - Test schema changes before pushing -2. **Descriptive migration messages** - Use clear, descriptive names for migrations -3. **Small, focused changes** - Avoid large schema changes in single migrations -4. **Review autogenerated migrations** - Don't blindly trust autogenerate output - -### For Reviews - -When reviewing PRs with database changes: - -1. **Check for migration files** - Ensure SQLModel changes include corresponding migrations -2. **Review migration logic** - Verify the migration does what it claims -3. **Consider data safety** - Check for potentially destructive operations -4. **Test migration reversal** - Ensure migrations can be safely downgraded - -## Troubleshooting - -### Common Issues - -**Issue**: "Environment variable POSTGRES_X not set" -- **Solution**: Ensure `.env.k8s.development` is properly configured with database credentials - -**Issue**: "Error creating temporary database" -- **Solution**: Verify PostgreSQL is running and credentials are correct - -**Issue**: "Error running existing migrations" -- **Solution**: Check that all migration files are valid and database is accessible - -**Issue**: False positive drift detection -- **Solution**: Ensure all SQLModel classes are imported in the migration env.py - -### Getting Help - -If you encounter issues with the schema drift detection system: - -1. Check the CI logs for specific error messages -2. Run the check locally for faster debugging -3. Verify your local database setup matches the expected configuration -4. Consult the team for complex migration scenarios - -## Technical Implementation - -### Script Architecture - -The `check_schema_drift.py` script follows this flow: - -```python -def main(): - # 1. Create temporary database - db_url = setup_test_database() - - try: - # 2. Apply existing migrations - run_existing_migrations(db_url) - - # 3. Check for new migrations needed - has_drift = check_for_new_migrations(db_url) - - # 4. Report results - if has_drift: - sys.exit(1) # Fail CI - else: - print("No drift detected") - - finally: - # 5. Clean up - cleanup_test_database(db_url) -``` - -### Security Considerations - -- Temporary databases use unique names to avoid conflicts -- Database credentials are loaded from environment variables -- Temporary resources are always cleaned up, even on failure -- No production data is accessed during drift detection - -This system ensures that database schema changes are always properly tracked through migrations, maintaining consistency across development, testing, and production environments. diff --git a/webui/src/docs/explanations.mdx b/webui/src/docs/explanations.mdx deleted file mode 100644 index 50612ad..0000000 --- a/webui/src/docs/explanations.mdx +++ /dev/null @@ -1,12 +0,0 @@ -# Explanations - -Explanations of how it works and why it works that way. - -## Architecture & Design -- [SmartEM Agent Design](/docs/explanations/smartem-agent-design) - Agent architecture -- [Backend-Agent Communication](/docs/explanations/backend-agent-communication-system-design) - Communication system design -- [EPU Data Structures](/docs/explanations/epu-data-structures) - EPU data format documentation -- [Technical Notes](/docs/explanations/technical-notes) - Technical implementation notes - -## Architecture Decision Records -- [Architecture Decisions](/docs/explanations/decisions) - All ADRs diff --git a/webui/src/docs/explanations/backend-agent-communication-system-design.mdx b/webui/src/docs/explanations/backend-agent-communication-system-design.mdx deleted file mode 100644 index 76a06f7..0000000 --- a/webui/src/docs/explanations/backend-agent-communication-system-design.mdx +++ /dev/null @@ -1,1048 +0,0 @@ -# Backend-to-Agent Communication System Design - -This document provides comprehensive system design documentation for the backend-to-agent communication system -implementing ADR #8's hybrid SSE + HTTP architecture. This system enables real-time delivery of microscope control -instructions from Kubernetes-hosted backend services to Windows workstations controlling -cryo-electron microscopes. - -## Architecture Overview - -### System Context - -The SmartEM Decisions platform operates in a distributed environment where backend services run in Kubernetes clusters -whilst agent services execute on Windows workstations directly connected to scientific equipment. -The communication system bridges this divide whilst meeting high-throughput requirements. - -**Implementation Status**: **COMPLETED** - This POC implementation provides a production-ready backend-to-agent -communication system with full SSE streaming, RabbitMQ integration, database persistence, and comprehensive -connection management. - -```mermaid -graph TB - subgraph k8s["Kubernetes Cluster - Diamond Light Source"] - subgraph packages["SmartEM Packages"] - subgraph backend["smartem_backend"] - api_server["api_server.py
FastAPI + SSE Endpoints"] - consumer["consumer.py
RabbitMQ Event Processing"] - conn_mgr["agent_connection_manager.py
Connection Health Monitoring"] - end - - subgraph common["smartem_common"] - schemas["schemas.py
Shared Data Models"] - utils["utils.py
Common Utilities"] - end - - subgraph athena["athena_api"] - athena_client["client.py
External API Integration"] - end - - subgraph mcp["smartem_mcp"] - mcp_server["server.py
Model Context Protocol"] - end - end - - subgraph infra["Infrastructure Services"] - db[("PostgreSQL
AgentSession, AgentInstruction
AgentConnection, AgentInstructionAcknowledgement")] - mq[("RabbitMQ
Event Communication
Instruction Lifecycle")] - end - end - - subgraph boundary["Network Boundary - Windows Workstations"] - subgraph agents["Agent Software (Windows)"] - subgraph agent_pkg["smartem_agent"] - fs_watcher["fs_watcher.py
File System Monitoring"] - fs_parser["fs_parser.py
EPU Data Parsing"] - sse_client["SSEAgentClient
Stream Connection"] - end - end - - subgraph equipment["Scientific Equipment"] - subgraph scope1["Microscope Workstation 1"] - epu1["EPU Software
(ThermoFisher)"] - em1["Cryo-EM Microscope 1"] - gpfs1["GPFS Storage
Image Data"] - end - - subgraph scope2["Microscope Workstation 2"] - epu2["EPU Software
(ThermoFisher)"] - em2["Cryo-EM Microscope 2"] - gpfs2["GPFS Storage
Image Data"] - end - - subgraph scopeN["Microscope Workstation N"] - epuN["EPU Software
(ThermoFisher)"] - emN["Cryo-EM Microscope N"] - gpfsN["GPFS Storage
Image Data"] - end - end - end - - %% Database and message queue connections - api_server --> db - api_server --> mq - consumer --> db - consumer --> mq - conn_mgr --> db - conn_mgr --> mq - - %% SSE streaming connections - api_server -.->|"SSE: /agent/{id}/session/{sid}/instructions/stream"| sse_client - - %% HTTP acknowledgement connections - sse_client -.->|"HTTP: /agent/{id}/session/{sid}/instructions/{iid}/ack"| api_server - - %% File system monitoring - fs_watcher --> fs_parser - fs_parser --> sse_client - - %% Equipment integration - epu1 --> gpfs1 - epu2 --> gpfs2 - epuN --> gpfsN - - fs_watcher -.->|"Monitor EPU Output"| gpfs1 - fs_watcher -.->|"Monitor EPU Output"| gpfs2 - fs_watcher -.->|"Monitor EPU Output"| gpfsN - - %% External API integration - consumer --> athena_client - - %% Package dependencies - backend -.-> common - agent_pkg -.-> common - fs_parser -.-> schemas - - %% Styling - classDef k8s fill:#e1f5fe,stroke:#0277bd,stroke-width:2px - classDef boundary fill:#fff3e0,stroke:#ef6c00,stroke-width:2px - classDef database fill:#f3e5f5,stroke:#7b1fa2,stroke-width:2px - classDef equipment fill:#e8f5e8,stroke:#2e7d32,stroke-width:2px - classDef communication fill:#fff8e1,stroke:#f57f17,stroke-width:2px - - class k8s,packages,backend,common,athena,mcp,infra k8s - class boundary,agents,agent_pkg,equipment boundary - class db,mq database - class scope1,scope2,scopeN,epu1,epu2,epuN,em1,em2,emN,gpfs1,gpfs2,gpfsN equipment - class api_server,sse_client,fs_watcher communication -``` - -### Service Architecture - -The communication system employs a **separate service approach** rather than integrating directly with the main API -service. This design provides: - -- **Isolation of concerns**: Communication logic remains separate from core business logic -- **Scalability independence**: Communication service can scale independently based on connection load -- **Operational simplicity**: Monitoring and debugging of persistent connections without affecting main API -- **Resource management**: Dedicated resources for managing long-lived SSE connections - -```mermaid -graph LR - subgraph services["Service Layer"] - main["Main API Service"] - comm["Communication Service"] - end - - subgraph data["Data Layer"] - db[("PostgreSQL")] - mq[("RabbitMQ")] - end - - subgraph agents["Agent Layer"] - agent["Agent Clients"] - end - - main --> db - main --> mq - comm --> db - comm --> mq - - mq --> comm - comm <--> agents - - main -.->|Events| mq - mq -.->|Instructions| comm -``` - -## Technical Stack Integration - -### FastAPI Integration - -The communication service leverages FastAPI's native SSE support through `StreamingResponse` and event streaming -patterns: - -```python -# Conceptual endpoint structure -@app.get("/agent/{agent_id}/instructions/stream") -async def stream_instructions(agent_id: str): - """SSE endpoint for streaming instructions to agents""" - -@app.post("/agent/{agent_id}/instructions/{instruction_id}/ack") -async def acknowledge_instruction(agent_id: str, instruction_id: str): - """HTTP endpoint for instruction acknowledgements""" -``` - -### RabbitMQ Message Flow - -The system integrates with the existing RabbitMQ infrastructure as an event communication backbone between ML components and the communication service: - -```mermaid -sequenceDiagram - participant ML as ML Pipeline - participant MQ as RabbitMQ - participant Comm as Communication Service - participant Agent as Agent Client - participant DB as PostgreSQL - - ML->>MQ: Publish instruction event - MQ->>Comm: Deliver to agent queue - Comm->>DB: Store instruction state - Comm->>Agent: Stream via SSE - Agent->>Comm: HTTP acknowledgement - Comm->>DB: Update delivery status - Comm->>MQ: Publish ACK event -``` - -### PostgreSQL Schema Design - -The communication system extends the existing database schema with instruction tracking tables: - -```sql --- Conceptual schema structure -CREATE TABLE agent_instructions ( - id UUID PRIMARY KEY, - agent_id VARCHAR NOT NULL, - instruction_type VARCHAR NOT NULL, - payload JSONB NOT NULL, - created_at TIMESTAMP WITH TIME ZONE NOT NULL, - delivered_at TIMESTAMP WITH TIME ZONE, - acknowledged_at TIMESTAMP WITH TIME ZONE, - status VARCHAR NOT NULL DEFAULT 'pending', - retry_count INTEGER DEFAULT 0 -); - -CREATE TABLE agent_connections ( - agent_id VARCHAR PRIMARY KEY, - connection_id VARCHAR NOT NULL, - connected_at TIMESTAMP WITH TIME ZONE NOT NULL, - last_heartbeat TIMESTAMP WITH TIME ZONE, - connection_type VARCHAR NOT NULL -- 'sse' -); -``` - -## Component Interactions and Data Flows - -### Primary Communication Flow (SSE) - -The primary communication path uses Server-Sent Events for efficient real-time instruction delivery: - -```mermaid -sequenceDiagram - participant Agent as Agent Client - participant Comm as Communication Service - participant MQ as RabbitMQ - participant DB as PostgreSQL - - Agent->>Comm: Establish SSE connection - Comm->>DB: Register connection - - loop Instruction Processing - MQ->>Comm: New instruction event - Comm->>DB: Store instruction - Comm->>Agent: Stream instruction (SSE) - Agent->>Comm: HTTP acknowledgement - Comm->>DB: Update delivery status - end - - Agent->>Comm: Connection closed - Comm->>DB: Clean up connection state -``` - - -### Error Handling and Recovery - -The system implements comprehensive error handling across multiple failure scenarios: - -```mermaid -graph TD - start([Instruction Generated]) --> sse{SSE Connected?} - - sse -->|Yes| stream[Stream via SSE] - sse -->|No| queue[Queue for Next Connection] - - stream --> ack{Acknowledgement Received?} - ack -->|Yes| complete[Mark Complete] - ack -->|No| retry{Retry Count < Max?} - - retry -->|Yes| delay[Exponential Backoff] - retry -->|No| failed[Mark Failed] - - delay --> reconnect[Reconnect SSE] - reconnect --> stream - - queue --> reconnect - failed --> end([End]) - complete --> end([End]) -``` - -## Scalability Design - -### Connection Management - -The system is designed to support **one session per agent machine** with a capacity of **20 concurrent SSE -connections**. This design aligns with the facility's requirements of up to 20 microscope workstations, where each workstation controls -a single microscope. - -```mermaid -graph TB - subgraph comm["Communication Service Instance"] - pool["Connection Pool"] - mgr["Connection Manager"] - health["Health Monitor"] - end - - subgraph agents["Agent Connections"] - agent1["Agent 1 (SSE)"] - agent2["Agent 2 (SSE)"] - agent3["Agent 3 (SSE)"] - agentN["Agent N (SSE)"] - end - - pool --> agent1 - pool --> agent2 - pool --> agent3 - pool --> agentN - - mgr --> pool - health --> pool - - pool -.->|Max 20 concurrent| limit[Connection Limit] -``` - -### Theoretical Scaling Limits - -**Current Architecture Bottlenecks:** - -1. **Database Write Performance**: High-frequency instruction persistence and state updates may impact database performance -2. **Database Connection Pool**: Connection pool limits for concurrent instruction storage and retrieval operations -3. **Memory Usage**: Each SSE connection maintains in-memory state (~1-2MB per connection) -4. **RabbitMQ Throughput**: Event notification capacity for real-time updates - -**Scaling Strategies:** - -- **Horizontal Scaling**: Deploy multiple communication service instances behind load balancer -- **Connection Sharding**: Distribute agents across service instances by agent ID hash -- **Resource Optimization**: Implement connection pooling and memory-efficient streaming -- **Database Optimization**: Use connection pooling and read replicas for instruction queries - -### Performance Characteristics - -**Expected Throughput:** -- **Instruction Frequency**: 1 instruction per 30-120 seconds per agent during active data collection -- **Peak Load**: 20 agents × 2 instructions/minute = 40 instructions/minute system-wide (at maximum frequency) -- **Message Size**: 1-10KB JSON payloads for microscope control instructions -- **Latency Requirements**: Sub-second delivery for real-time workflow efficiency - -## Implementation Specifications - -### SSE Streaming Service Design - -The SSE streaming service implements persistent connections with automatic reconnection handling: - -```python -# Conceptual implementation structure -class SSEInstructionStream: - """Manages SSE connections and instruction streaming""" - - async def stream_instructions(self, agent_id: str) -> AsyncIterator[str]: - """Async generator for SSE instruction stream""" - - async def handle_connection_lifecycle(self, agent_id: str): - """Manages connection establishment, maintenance, and cleanup""" - - async def process_pending_instructions(self, agent_id: str): - """Retrieves and processes pending instructions from database for SSE delivery""" - -class ConnectionManager: - """Manages active SSE connections and health monitoring""" - - def register_connection(self, agent_id: str, connection_id: str): - """Register new SSE connection""" - - def cleanup_connection(self, agent_id: str): - """Clean up disconnected SSE connection""" - - async def health_check_connections(self): - """Monitor connection health and handle failures""" -``` - -### HTTP Acknowledgement Endpoints - -HTTP acknowledgement endpoints provide reliable delivery confirmation: - -```python -# Conceptual acknowledgement handling -@dataclass -class InstructionAcknowledgement: - instruction_id: str - agent_id: str - acknowledged_at: datetime - status: Literal["received", "processed", "failed"] - error_message: str | None = None - -class AcknowledgementHandler: - """Handles instruction acknowledgements and delivery tracking""" - - async def process_acknowledgement( - self, - ack: InstructionAcknowledgement - ) -> bool: - """Process and store instruction acknowledgement""" - - async def handle_declined_instruction( - self, - agent_id: str, - instruction_id: str, - reason: str - ): - """Handle agent declining instruction execution""" -``` - -### SSE Retry Implementation - -The system implements robust SSE reconnection with exponential backoff: - -```python -class SSERetryManager: - """Manages SSE connection retry logic with exponential backoff""" - - def should_retry(self, agent_id: str, attempt_count: int) -> bool: - """Determine if SSE connection should be retried""" - - def calculate_backoff_delay(self, attempt_count: int) -> int: - """Calculate exponential backoff delay for reconnection""" - - async def reconnect_with_backoff(self, agent_id: str) -> bool: - """Attempt SSE reconnection with backoff delay""" -``` - -## Implementation Status & Components - -This POC implementation provides a complete working system with the following implemented components: - -### Completed Features - -#### 1. Database Schema & Migration (Alembic) -- **AgentSession**: Session management for agent connections -- **AgentInstruction**: Instruction storage with metadata and lifecycle tracking -- **AgentConnection**: Real-time connection tracking with heartbeat monitoring -- **AgentInstructionAcknowledgement**: Comprehensive acknowledgement tracking - -#### 2. FastAPI SSE Endpoints -- **`/agent/{agent_id}/session/{session_id}/instructions/stream`**: Real-time SSE streaming -- **`/agent/{agent_id}/session/{session_id}/instructions/{instruction_id}/ack`**: HTTP acknowledgement -- **Debug endpoints**: Connection statistics and session management - -#### 3. RabbitMQ Integration -- **Event Publishers**: Agent instruction lifecycle events -- **Consumer Handlers**: Process instruction events and database updates -- **Message Types**: `agent.instruction.created`, `agent.instruction.updated`, `agent.instruction.expired` - -#### 4. Enhanced Agent Client (`SSEAgentClient`) -- **Exponential backoff retry logic** with jitter -- **Connection statistics and monitoring** -- **Comprehensive error handling and recovery** -- **Processing time tracking for performance metrics** - -#### 5. Connection Management Service (`AgentConnectionManager`) -- **Automatic stale connection cleanup** (2-minute timeout) -- **Instruction expiration handling** with retry logic -- **Session activity monitoring** (1-hour inactivity threshold) -- **Real-time statistics and health monitoring** - -#### 6. Production-Ready Example Client -- **Complete instruction processing workflow** -- **Multiple instruction type support** (stage movement, image acquisition) -- **Processing time measurement and acknowledgement** -- **Enhanced error handling and statistics display** - -### 🚀 Key Implementation Highlights - -- **Database-backed persistence**: All instruction state persisted with full audit trail -- **Connection resilience**: Automatic reconnection with exponential backoff -- **Health monitoring**: Background tasks for cleanup and monitoring -- **Production logging**: Comprehensive logging at all system levels -- **Type safety**: Full Pydantic model validation throughout -- **Test-friendly design**: Debug endpoints for system verification - -## Message Lifecycle Management - -### Sequential Delivery Requirements - -The system ensures **sequential message delivery** to maintain microscope control instruction ordering: - -```mermaid -stateDiagram-v2 - [*] --> Generated: Instruction created - Generated --> Queued: Added to agent queue - Queued --> Streaming: SSE connection available - Queued --> Retry: Connection unavailable - - Streaming --> Delivered: Agent receives via SSE - Retry --> Queued: After backoff delay - - Delivered --> Acknowledged: Agent confirms receipt - Delivered --> Declined: Agent declines execution - Delivered --> Timeout: No acknowledgement received - - Acknowledged --> [*]: Complete - Declined --> [*]: Logged and complete - Timeout --> Retry: Attempt redelivery - - Retry --> Queued: Requeue instruction - Retry --> Failed: Max retries exceeded - - Failed --> [*]: Mark as failed -``` - -### Database Persistence Strategy - -The system uses **PostgreSQL as source of truth** with RabbitMQ as the event communication backbone: - -```mermaid -graph LR - subgraph truth["Source of Truth"] - db[("PostgreSQL")] - end - - subgraph events["Event Communication"] - mq[("RabbitMQ")] - end - - subgraph ops["Operational Queries"] - queries["Instruction State
Connection Health
Performance Metrics
Audit Trails"] - end - - db --> mq - db --> queries - - db -.->|Primary| primary[Instruction Persistence
State Management
Audit Logging] - mq -.->|Secondary| secondary[Event Notification
Component Communication
Real-time Updates] -``` - -### Agent Restart Message Replay (TODO) - -**Current Status**: Not implemented - marked as future requirement - -**Design Considerations**: -- Determine replay window (e.g., last 24 hours of unacknowledged instructions) -- Handle duplicate instruction detection and prevention -- Manage instruction sequence numbering across agent restarts -- Implement replay request mechanism from agent on startup - - -## External Message Types and Transformation Pipeline - -### External Message Types - -The system processes external RabbitMQ messages from data processing pipelines and machine learning components -to trigger real-time decision-making and microscope control instructions. These external messages represent -completion events from various stages of the cryo-EM data processing workflow: - -**Primary External Message Types:** -- **MOTION_CORRECTION_COMPLETE**: Indicates completion of motion correction processing for collected images -- **CTF_COMPLETE**: Signals completion of contrast transfer function (CTF) estimation for image quality assessment -- **PARTICLE_PICKING_COMPLETE**: Notifies completion of automated particle identification in micrographs -- **PARTICLE_SELECTION_COMPLETE**: Indicates completion of particle quality assessment and selection -- **GRIDSQUARE_MODEL_PREDICTION**: Provides machine learning predictions for gridsquare quality and suitability -- **FOILHOLE_MODEL_PREDICTION**: Delivers ML predictions for individual foilhole targeting recommendations -- **MODEL_PARAMETER_UPDATE**: Communicates updates to ML model parameters affecting decision thresholds - -These external messages contain scientific data processing results, quality metrics, and ML predictions that drive -the backend's decision logic for microscope control instructions. - -### Message Transformation Pipeline - -The transformation pipeline converts external data processing events into actionable SSE instructions for agents. -This pipeline operates within the `consumer.py` component and implements the core business logic for scientific -decision-making: - -```mermaid -graph TB - subgraph external["External Systems"] - ml_pipeline["ML Pipeline"] - image_proc["Image Processing"] - ctf_est["CTF Estimation"] - particle_pick["Particle Picking"] - end - - subgraph rabbitmq["Message Queue"] - ext_queue["External Event Queue"] - end - - subgraph backend["Backend Processing"] - consumer["consumer.py"] - decision_logic["Decision Logic Engine"] - threshold_eval["Threshold Evaluation"] - instruction_gen["Instruction Generator"] - end - - subgraph communication["Communication System"] - sse_stream["SSE Instruction Stream"] - agent_client["Agent Clients"] - end - - subgraph microscope["Microscope Control"] - athena_api["Athena API"] - epu_control["EPU Software"] - end - - ml_pipeline --> ext_queue - image_proc --> ext_queue - ctf_est --> ext_queue - particle_pick --> ext_queue - - ext_queue --> consumer - consumer --> decision_logic - decision_logic --> threshold_eval - threshold_eval --> instruction_gen - - instruction_gen --> sse_stream - sse_stream --> agent_client - agent_client --> athena_api - athena_api --> epu_control -``` - -**Transformation Process:** - -1. **Message Reception**: External messages arrive via dedicated RabbitMQ queues with processing results -2. **Data Extraction**: Consumer extracts quality metrics, coordinates, and prediction values from message payloads -3. **Decision Logic Application**: Business rules evaluate data against configurable quality thresholds -4. **Instruction Generation**: Decision outcomes generate specific microscope control instructions -5. **Traceability Injection**: Instructions include metadata linking back to originating external messages -6. **SSE Delivery**: Generated instructions are queued for real-time delivery to appropriate agents - -**Quality Threshold Examples:** -```json -{ - "gridsquare_quality_threshold": 0.7, - "foilhole_ice_thickness_max": 150.0, - "ctf_resolution_minimum": 4.0, - "motion_correction_drift_max": 2.0 -} -``` - -### Athena API Control Instructions - -The system generates specific instruction types that correspond to Athena API control capabilities for microscope -workflow management. These instructions represent the actionable outputs of the decision-making process: - -**Primary Control Instruction Types:** - -#### athena.control.reorder_foilholes -Reorders foilhole acquisition sequence based on ML predictions and quality assessments: - -```json -{ - "instruction_id": "uuid-v4", - "instruction_type": "athena.control.reorder_foilholes", - "version": "1.0", - "timestamp": "2025-09-23T10:30:00Z", - "payload": { - "gridsquare_id": "GS_001_002", - "foilhole_order": [ - {"foilhole_id": "FH_001", "priority_score": 0.95}, - {"foilhole_id": "FH_003", "priority_score": 0.87}, - {"foilhole_id": "FH_002", "priority_score": 0.72} - ], - "reorder_reason": "ml_prediction_update" - }, - "metadata": { - "session_id": "session-uuid", - "originating_message_id": "external-msg-uuid", - "decision_timestamp": "2025-09-23T10:29:45Z", - "quality_threshold": 0.7 - } -} -``` - -#### athena.control.reorder_gridsquares -Reorders gridsquare acquisition sequence based on overall quality assessments: - -```json -{ - "instruction_id": "uuid-v4", - "instruction_type": "athena.control.reorder_gridsquares", - "version": "1.0", - "timestamp": "2025-09-23T10:35:00Z", - "payload": { - "gridsquare_order": [ - {"gridsquare_id": "GS_001_003", "quality_score": 0.92}, - {"gridsquare_id": "GS_001_001", "quality_score": 0.88}, - {"gridsquare_id": "GS_001_002", "quality_score": 0.74} - ], - "reorder_strategy": "quality_optimised" - }, - "metadata": { - "session_id": "session-uuid", - "originating_message_id": "gridsquare-prediction-uuid", - "model_version": "v2.1.0", - "prediction_confidence": 0.89 - } -} -``` - -#### athena.control.skip_gridsquares -Skips gridsquares that fail to meet quality thresholds: - -```json -{ - "instruction_id": "uuid-v4", - "instruction_type": "athena.control.skip_gridsquares", - "version": "1.0", - "timestamp": "2025-09-23T10:40:00Z", - "payload": { - "gridsquares_to_skip": [ - { - "gridsquare_id": "GS_001_005", - "skip_reason": "quality_below_threshold", - "quality_score": 0.45 - }, - { - "gridsquare_id": "GS_001_007", - "skip_reason": "ice_contamination_detected", - "contamination_level": 0.85 - } - ], - "skip_strategy": "quality_based" - }, - "metadata": { - "session_id": "session-uuid", - "originating_message_id": "quality-assessment-uuid", - "threshold_applied": 0.6, - "assessment_method": "ml_classification" - } -} -``` - -### Complete Message Flow Diagram - -The following diagram illustrates the complete message flow from external systems through to microscope control: - -```mermaid -sequenceDiagram - participant ExtSys as External Systems
(ML Pipeline, Image Processing) - participant MQ as RabbitMQ
(External Events) - participant Consumer as Backend Consumer
(consumer.py) - participant DB as PostgreSQL
(Instruction Storage) - participant SSE as SSE Stream
(Real-time Delivery) - participant Agent as Agent Client
(Windows Workstation) - participant Athena as Athena API
(Microscope Control) - participant EPU as EPU Software
(ThermoFisher) - - Note over ExtSys: Data processing completes - ExtSys->>MQ: MOTION_CORRECTION_COMPLETE
CTF_COMPLETE
PARTICLE_PICKING_COMPLETE
ML_PREDICTION - - MQ->>Consumer: External event delivery - - Note over Consumer: Message transformation pipeline - Consumer->>Consumer: Extract quality metrics
Apply decision thresholds
Generate control instructions - - Consumer->>DB: Store instruction with traceability
(originating_message_id) - Consumer->>MQ: Publish instruction event - - MQ->>SSE: Instruction ready for delivery - SSE->>Agent: Stream instruction via SSE
(athena.control.*) - - Agent->>SSE: HTTP acknowledgement - SSE->>DB: Update delivery status - - Note over Agent: Process microscope control instruction - Agent->>Athena: Execute control command
(reorder_foilholes, skip_gridsquares) - - Athena->>EPU: Apply workflow changes - EPU->>Agent: Execution confirmation - - Agent->>SSE: Final execution status - SSE->>DB: Update instruction completion - - Note over DB: Complete audit trail
External message → Decision → Execution -``` - -**Key Message Flow Characteristics:** - -1. **Scientific Traceability**: Every instruction maintains linkage to originating external messages for reproducibility -2. **Real-time Processing**: Sub-second transformation from external events to microscope control instructions -3. **Quality-driven Decisions**: Business logic applies configurable thresholds to scientific data -4. **Comprehensive Audit Trail**: Full tracking from data processing results through to microscope actions -5. **Failure Recovery**: Instruction replay capability maintains workflow continuity during system interruptions - -**Message Transformation Examples:** - -- **CTF_COMPLETE** with resolution < 4.0Å → **athena.control.skip_gridsquares** (poor quality) -- **GRIDSQUARE_MODEL_PREDICTION** with score > 0.8 → **athena.control.reorder_gridsquares** (prioritise high-quality) -- **FOILHOLE_MODEL_PREDICTION** with updated rankings → **athena.control.reorder_foilholes** (optimise sequence) - -This transformation pipeline ensures that scientific data processing results directly drive microscope control decisions -with full traceability and audit capabilities for research reproducibility. - -## Extensibility Design - -### JSON Message Vocabulary - -The system implements **comprehensive JSON message vocabulary** for microscope control instructions: - -**Core Message Structure:** -```json -{ - "instruction_id": "uuid-v4", - "instruction_type": "athena.control.*", - "version": "1.0", - "timestamp": "2025-09-23T10:30:00Z", - "payload": { - // Instruction-specific data - }, - "metadata": { - "session_id": "session-uuid", - "originating_message_id": "external-msg-uuid", - "decision_timestamp": "2025-09-23T10:29:45Z", - "quality_threshold": 0.7 - } -} -``` - -**Implemented Instruction Types:** -- `athena.control.reorder_foilholes`: Reorder foilhole acquisition sequence based on ML predictions -- `athena.control.reorder_gridsquares`: Reorder gridsquare acquisition sequence based on quality assessment -- `athena.control.skip_gridsquares`: Skip gridsquares that fail quality thresholds - -**Enhanced Metadata for Scientific Traceability:** -- `originating_message_id`: Links instruction back to external processing event -- `decision_timestamp`: Records when decision logic was applied -- `quality_threshold`: Documents threshold values used in decision-making -- `model_version`: Tracks ML model version for reproducibility -- `prediction_confidence`: Records confidence level of ML predictions - -**Extensibility Features**: -- Version field for message schema evolution across instruction types -- Flexible payload structure accommodating diverse microscope control requirements -- Enhanced metadata section supporting scientific traceability and reproducibility -- Type-safe instruction validation using Pydantic models for data integrity -- Originating message linkage for complete audit trails from data processing to execution -- Configurable quality thresholds enabling adaptive decision-making - -### Implemented ML Integration - -The architecture provides **production machine learning and data processing integration**: - -```mermaid -graph TB - subgraph ml["ML Pipeline Integration (Implemented)"] - model["Prediction Models
(Gridsquare/Foilhole)"] - pipeline["Processing Pipeline
(Motion/CTF/Particles)"] - feedback["Execution Feedback
(Future Enhancement)"] - end - - subgraph comm["Communication System"] - consumer["Message Consumer
(consumer.py)"] - decision["Decision Logic Engine"] - instructions["Instruction Generation"] - service["SSE Communication Service"] - end - - subgraph agents["Agent Layer"] - agent["Agent Execution"] - athena["Athena API Integration"] - results["Execution Results"] - end - - model --> pipeline - pipeline --> consumer - consumer --> decision - decision --> instructions - instructions --> service - service --> agent - agent --> athena - athena --> results - results -.->|Future| feedback - feedback -.->|Future| model - - classDef implemented fill:#d4edda,stroke:#155724,stroke-width:2px - classDef future fill:#fff3cd,stroke:#856404,stroke-width:2px - - class model,pipeline,consumer,decision,instructions,service,agent,athena,results implemented - class feedback future -``` - -**Current ML Integration Features:** -- **Real-time Processing Results**: Integration with motion correction, CTF estimation, and particle picking pipelines -- **ML Model Predictions**: Gridsquare and foilhole quality prediction integration -- **Quality-driven Decisions**: Automated decision-making based on configurable quality thresholds -- **Scientific Traceability**: Full audit trail from ML predictions to microscope actions - -**Future Enhancement Areas:** -- **Execution Feedback Loop**: Collection of execution results to improve ML model predictions -- **Adaptive Thresholds**: Dynamic threshold adjustment based on session performance -- **Predictive Analytics**: Advanced analytics for experiment outcome prediction - -## Traceability and Monitoring - -### Message Tracking System - -The system provides **full message tracking** with comprehensive identification: - -```mermaid -graph LR - subgraph tracking["Message Tracking"] - id["Instruction ID (UUID)"] - origin["Origin Timestamp"] - causation["Causation Chain"] - status["Delivery Status"] - end - - subgraph audit["Audit Trail"] - created["Creation Event"] - queued["Queue Event"] - delivered["Delivery Event"] - acked["Acknowledgement Event"] - end - - tracking --> audit - - subgraph queries["Operational Queries"] - pending["Pending Instructions"] - failed["Failed Deliveries"] - performance["Performance Metrics"] - end - - audit --> queries -``` - -### Monitoring Architecture - -```mermaid -graph TB - subgraph metrics["Metrics Collection"] - conn["Connection Metrics"] - delivery["Delivery Metrics"] - performance["Performance Metrics"] - errors["Error Rates"] - end - - subgraph monitoring["Monitoring Stack"] - prometheus["Prometheus"] - grafana["Grafana Dashboards"] - alerts["Alert Manager"] - end - - subgraph logs["Logging"] - structured["Structured Logs"] - correlation["Correlation IDs"] - levels["Log Levels"] - end - - metrics --> monitoring - logs --> monitoring -``` - -## Operational Considerations - -### SSE Connection Health Monitoring - -**Health Check Mechanisms**: -- Periodic heartbeat messages via SSE stream -- Connection timeout detection and cleanup -- Automatic reconnection attempts with exponential backoff -- Connection state synchronisation with database - -**Monitoring Metrics**: -- Active connection count per agent -- Connection duration and stability -- Reconnection frequency and success rates -- Memory usage per connection - -### Debugging Message Delivery Issues - -**Debugging Capabilities**: -```python -# Conceptual debugging interface -class DeliveryDebugger: - """Tools for debugging message delivery issues""" - - def trace_instruction_lifecycle(self, instruction_id: str) -> DeliveryTrace: - """Trace complete instruction delivery lifecycle""" - - def diagnose_connection_issues(self, agent_id: str) -> ConnectionDiagnosis: - """Diagnose SSE connection problems""" - - def analyse_delivery_patterns(self, agent_id: str, timeframe: timedelta) -> Analysis: - """Analyse delivery success patterns for optimization""" -``` - -**Troubleshooting Features**: -- Real-time delivery status dashboard -- Instruction replay capability for testing -- Connection diagnostics with detailed error reporting -- Performance profiling for bottleneck identification - -### Resource Management - -**Connection Resource Management**: -- Concurrent connection capacity (20 connections for facility requirements) -- Connection memory usage monitoring and alerting -- Automatic connection cleanup on agent disconnect -- Resource pool management for database connections - -**Performance Optimization**: -- Connection keep-alive optimization for long-lived SSE streams -- Message batching for high-frequency instruction bursts -- Database query optimization for instruction retrieval -- Memory-efficient JSON streaming for large instruction payloads - -## Critical Analysis and Risk Assessment - -### Potential Bottlenecks - -**Identified Bottlenecks**: - -1. **Database Write Performance**: High-frequency instruction persistence and state updates may impact database performance under load -2. **Database Connection Contention**: Concurrent access from multiple service instances may strain database connection pools -3. **Memory Usage Growth**: Long-lived SSE connections accumulate memory usage over time -4. **Single Point of Failure**: Communication service represents single point of failure for all agents - -**Mitigation Strategies**: -- Implement horizontal scaling with load balancing for connection distribution -- Use database connection pooling and async operations for performance -- Implement memory leak detection and connection lifecycle management -- Deploy redundant service instances with automatic failover capabilities - -### Integration Complexities - -**Architectural Challenges**: - -1. **RabbitMQ-SSE Bridge**: Complex event bridging between message queues and SSE streams -2. **State Synchronisation**: Maintaining consistency between RabbitMQ, PostgreSQL, and SSE connection state -3. **Error Propagation**: Ensuring error conditions propagate correctly across system boundaries -4. **Testing Complexity**: Integration testing across multiple protocols and failure scenarios - -**Recommended Approaches**: -- Implement comprehensive integration testing with realistic failure simulation -- Use event sourcing patterns for consistent state management -- Deploy canary deployments for safe production rollouts -- Establish clear error handling contracts between components - -### Monitoring and Operational Gaps - -**Identified Gaps**: - -1. **End-to-End Tracing**: Limited visibility into complete instruction delivery lifecycle -2. **Capacity Planning**: Insufficient metrics for predicting scaling requirements -3. **Agent Health Correlation**: Limited correlation between agent health and delivery success - -**Required Improvements**: -- Implement distributed tracing with correlation IDs across all components -- Develop capacity planning dashboards with predictive analytics -- Create agent health correlation dashboards for operational insights - -This system design provides a robust foundation for real-time backend-to-agent communication whilst maintaining the -flexibility for future enhancements and scaling requirements in the SmartEM Decisions scientific computing platform. diff --git a/webui/src/docs/explanations/decisions.mdx b/webui/src/docs/explanations/decisions.mdx deleted file mode 100644 index 75b127d..0000000 --- a/webui/src/docs/explanations/decisions.mdx +++ /dev/null @@ -1,19 +0,0 @@ -# Architectural Decision Records - -Architectural decisions are made throughout a project's lifetime. As a way of keeping track of these decisions, we record these decisions in Architecture Decision Records (ADRs) listed below. - -## ADRs - -- [ADR-0001: Record Architecture Decisions](/docs/explanations/decisions/0001-record-architecture-decisions) -- [ADR-0002: Switched to Python Copier Template (superseded by ADR-0011)](/docs/explanations/decisions/0002-switched-to-python-copier-template) -- [ADR-0003: Message Queue Message Grouping](/docs/explanations/decisions/0003-message-queue-message-grouping) -- [ADR-0004: Zocalo Dependency-Free](/docs/explanations/decisions/0004-zocalo-dependency-free) -- [ADR-0005: Detect Secrets for Secret Scanning](/docs/explanations/decisions/0005-detect-secrets-for-secret-scanning) -- [ADR-0006: Sealed Secrets for Kubernetes](/docs/explanations/decisions/0006-sealed-secrets-kubernetes-secrets-management) -- [ADR-0007: Eliminate SmartEM API Circular Dependency](/docs/explanations/decisions/0007-eliminate-smartem-api-circular-dependency) -- [ADR-0008: Backend to Agent Communication Architecture](/docs/explanations/decisions/0008-backend-to-agent-communication-architecture) -- [ADR-0009: Commit Generated Route Tree](/docs/explanations/decisions/0009-commit-generated-route-tree) -- [ADR-0010: Use Shiki for Syntax Highlighting](/docs/explanations/decisions/0010-shiki-syntax-highlighting) -- [ADR-0011: Remove Python Copier Template](/docs/explanations/decisions/0011-remove-python-copier-template) - -For more information on ADRs see this [blog by Michael Nygard](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions). diff --git a/webui/src/docs/explanations/decisions/0001-record-architecture-decisions.mdx b/webui/src/docs/explanations/decisions/0001-record-architecture-decisions.mdx deleted file mode 100644 index 1ce4d3b..0000000 --- a/webui/src/docs/explanations/decisions/0001-record-architecture-decisions.mdx +++ /dev/null @@ -1,19 +0,0 @@ -# 1. Record architecture decisions - -## Status - -Accepted - -## Context - -We need to record the architectural decisions made on this project. - -## Decision - -We will use Architecture Decision Records, as [described by Michael -Nygard](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions). - -## Consequences - -See Michael Nygard's article, linked above. To create new ADRs we will copy and -paste from existing ones. diff --git a/webui/src/docs/explanations/decisions/0002-switched-to-python-copier-template.mdx b/webui/src/docs/explanations/decisions/0002-switched-to-python-copier-template.mdx deleted file mode 100644 index c430699..0000000 --- a/webui/src/docs/explanations/decisions/0002-switched-to-python-copier-template.mdx +++ /dev/null @@ -1,28 +0,0 @@ -# 2. Adopt python-copier-template for project structure - -## Status - -Superseded by ADR-0011 - -> **Note**: This decision was reversed on 2026-01-05. See [ADR-0011](/docs/explanations/decisions/0011-remove-python-copier-template) for details. - -## Context - -We should use the following [python-copier-template](https://github.com/DiamondLightSource/python-copier-template). -The template will ensure consistency in developer environments and package management. - -## Decision - -We have switched to using the template. - -## Consequences - -This module will use a fixed set of tools as developed in `python-copier-template` and can pull from this -template to update the packaging to the latest techniques. - -As such, the developer environment may have changed, the following could be different: - -- linting -- formatting -- pip venv setup -- CI/CD diff --git a/webui/src/docs/explanations/decisions/0003-message-queue-message-grouping.mdx b/webui/src/docs/explanations/decisions/0003-message-queue-message-grouping.mdx deleted file mode 100644 index c3e9d53..0000000 --- a/webui/src/docs/explanations/decisions/0003-message-queue-message-grouping.mdx +++ /dev/null @@ -1,22 +0,0 @@ -# 3. Short descriptive title - -Date: 28/10/2024 - -## Status - -In Progress - -## Context - -Do we want to have separate queues for different types of processing data feedback which will lead to -different method calls (i.e. CTF results vs. particle picking results)? - -The other option would be to have the messages contain something to indicate the relevant callback method. - -## Decision - -What decision we made. - -## Consequences - -What we will do as a result of this decision. diff --git a/webui/src/docs/explanations/decisions/0004-zocalo-dependency-free.mdx b/webui/src/docs/explanations/decisions/0004-zocalo-dependency-free.mdx deleted file mode 100644 index ea42faa..0000000 --- a/webui/src/docs/explanations/decisions/0004-zocalo-dependency-free.mdx +++ /dev/null @@ -1,30 +0,0 @@ -# 3. No dependency on python-zocalo - -Date: 08/11/2024 - -## Status - -Accepted - -## Context - -We initially needed to send logs to Graylog, and this functionality was already implemented in `python-zocalo`. -Hence, one option was to import any logging-related functionality from Zocalo, while another option was to just -copy the code across, duplicating it in our repo. - -**Update**: Graylog functionality has since been removed from the project. In production Kubernetes, all stdout and -stderr are automatically sent to a managed Graylog instance, so we don't need to handle this in our application -code. For local development, we now use standard Python logging to keep things simple. - -## Decision - -According to Dan Hatton: - -> I was intending to keep it free of the zocalo dependency if possible because there are internal -> discussions about whether we move off of it longer term - -## Consequences - -We will stay free of any Zocalo dependencies. Initially we duplicated logging functionality within this project, -but have since simplified to use standard Python logging as production Kubernetes handles log aggregation -automatically. diff --git a/webui/src/docs/explanations/decisions/0005-detect-secrets-for-secret-scanning.mdx b/webui/src/docs/explanations/decisions/0005-detect-secrets-for-secret-scanning.mdx deleted file mode 100644 index d6cbd61..0000000 --- a/webui/src/docs/explanations/decisions/0005-detect-secrets-for-secret-scanning.mdx +++ /dev/null @@ -1,52 +0,0 @@ -# 5. Use detect-secrets for secret scanning - -Date: 21/08/2025 - -## Status - -Accepted - -## Context - -The SmartEM Decisions project requires robust secret scanning to protect sensitive research data, database -credentials, API keys, and Kubernetes cluster secrets. As part of the Diamond Light Source facility infrastructure, -high security standards are essential whilst supporting scientific computing workflows. - -The development team evaluated secret scanning tools for integration into the existing sophisticated pre-commit and -CI/CD pipeline (Python 3.12+, ruff, pyright). The organisational cybersecurity team recommended Gitleaks for -standardisation across projects. - -Key requirements included: -- Integration with Python 3.12+ ecosystem and existing toolchain -- Handling scientific computing patterns (chemical formulas, gene sequences, scientific notation) without excessive - false positives -- Support for high-throughput processing without workflow disruption -- Enterprise-grade baseline management for research environments - -Three tools were evaluated: -- **Gitleaks**: High-performance Go implementation, organisational preference, but higher false positives in - scientific contexts -- **TruffleHog**: Advanced entropy analysis, but resource-intensive with SaaS dependencies -- **detect-secrets**: Python-native, superior false positive handling, sophisticated baseline management - -## Decision - -We will use **detect-secrets** as the primary secret scanning tool, integrated into both pre-commit hooks and -CI/CD pipelines, despite the organisational preference for Gitleaks standardisation. - -## Consequences - -**Positive:** -- Native Python integration with existing development workflow -- Superior false positive management for scientific computing patterns -- Enterprise-grade baseline system for managing known safe patterns -- Faster CI/CD execution through incremental scanning approach -- Flexible plugin architecture for research-specific customisation - -**Negative:** -- Divergence from organisational tooling standardisation -- Potential knowledge silos between teams using different tools -- Responsibility for maintaining tool-specific expertise within the team - -**Mitigation:** -- Comprehensive documentation of configuration and workflows diff --git a/webui/src/docs/explanations/decisions/0006-sealed-secrets-kubernetes-secrets-management.mdx b/webui/src/docs/explanations/decisions/0006-sealed-secrets-kubernetes-secrets-management.mdx deleted file mode 100644 index 254e199..0000000 --- a/webui/src/docs/explanations/decisions/0006-sealed-secrets-kubernetes-secrets-management.mdx +++ /dev/null @@ -1,73 +0,0 @@ -# 6. Use Sealed Secrets for Kubernetes secrets management - -Date: 22/08/2025 - -## Status - -Accepted - -## Context - -The SmartEM Decisions project requires secure secrets management for Kubernetes deployments across development, staging, -and production environments. The current approach of committing plain secrets.yaml files with test passwords to version -control presents security risks and violates organisational security standards at Diamond Light Source. - -The project deploys to three environments using local development machines without CI/CD automation. Secrets include -PostgreSQL database credentials, RabbitMQ message queue authentication, and other sensitive configuration values -required for the scientific computing workflow. - -The Diamond Light Source Kubernetes team recommended Sealed Secrets for organisational standardisation, referencing -their internal development portal guide (https://dev-portal.diamond.ac.uk/guide/kubernetes/tutorials/secrets/) for -implementation guidance. - -Five approaches were evaluated for secrets management: - -1. **Kustomize Secret Generators**: Generate secrets at build time using kustomize secretGenerator functionality -2. **External Secrets Operator**: Pull secrets from external systems such as HashiCorp Vault or AWS Secrets Manager -3. **Sealed Secrets**: Encrypt secrets using asymmetric cryptography that can be safely committed to version control -4. **Environment-Specific .env Files**: Use gitignored .env files with generation scripts for each environment -5. **Development Tools Integration**: Generate secrets dynamically within the existing dev-k8s.sh deployment script - -Key requirements included: -- Eliminate plain text secrets from version control -- Support local deployment workflow without CI/CD infrastructure -- Maintain version control history for secret management -- Align with Diamond Light Source organisational standards -- Support secret rotation and environment-specific values - -## Decision - -We will use **Sealed Secrets** as the primary Kubernetes secrets management solution, implemented through the -sealed-secrets controller and kubeseal CLI tool. - -Sealed Secrets uses asymmetric cryptography where secrets are encrypted with a public key and can only be decrypted by -the sealed-secrets controller running in the target Kubernetes cluster. This allows encrypted SealedSecret resources to -be safely committed to version control whilst maintaining security. - -Implementation will include: -- Installation of sealed-secrets controller in each Kubernetes environment -- Integration with existing deployment workflow via tools/k8s/generate-sealed-secrets.sh script -- Documentation in docs/how-to/manage-kubernetes-secrets.md for team procedures -- Environment-specific encryption keys for development, staging, and production clusters - -## Consequences - -**Positive:** -- Eliminates security risk of committing plain text secrets to version control -- Aligns with Diamond Light Source organisational standards and team recommendations -- Enables version control tracking of secret changes whilst maintaining security -- Supports local deployment model without requiring external secret management infrastructure -- Provides secret rotation capabilities with audit trail through Git history -- Integrates seamlessly with existing Kubernetes deployment workflow - -**Negative:** -- Introduces learning curve for team members unfamiliar with sealed-secrets tooling -- Creates dependency on sealed-secrets controller availability in each cluster environment -- Requires careful management of sealed-secrets private keys for cluster recovery scenarios -- Additional complexity compared to plain Kubernetes secrets for local development workflows - -**Implementation considerations:** -- Backup and secure storage of sealed-secrets private keys for disaster recovery -- Documentation of secret rotation procedures for team knowledge sharing -- Integration testing to ensure sealed-secrets work correctly across all three environments -- Training for team members on kubeseal CLI usage and troubleshooting procedures diff --git a/webui/src/docs/explanations/decisions/0007-eliminate-smartem-api-circular-dependency.mdx b/webui/src/docs/explanations/decisions/0007-eliminate-smartem-api-circular-dependency.mdx deleted file mode 100644 index 2858ee3..0000000 --- a/webui/src/docs/explanations/decisions/0007-eliminate-smartem-api-circular-dependency.mdx +++ /dev/null @@ -1,77 +0,0 @@ -# 7. Eliminate circular dependency between smartem_api and smartem_backend - -Date: 23/08/2025 - -## Status - -Accepted - -## Context - -The SmartEM Decisions project developed a circular dependency between the `smartem_api` and `smartem_backend` packages -that violated clean architecture principles and created maintenance challenges for the scientific computing workflow -automation system. - -The circular dependency manifested in two ways: -1. **Package-level dependency**: `smartem_backend` declared dependency on `smartem_api` in its pyproject.toml - configuration, requiring the API package for core backend functionality -2. **Module-level imports**: `smartem_api.server` module imported extensively from `smartem_backend` components - including database models, message queue publishers, and utility functions - -This circular relationship created several problems for the cryo-electron microscopy workflow system: -- **Build complexity**: Package installation order became critical and error-prone during development setup -- **Architectural violation**: Backend logic was split across packages in non-intuitive ways, complicating system - understanding -- **Maintenance burden**: Changes to core backend functionality required coordinated updates across multiple packages -- **Import confusion**: Developers struggled to understand which package provided specific functionality -- **Testing complexity**: Circular imports made unit testing and mocking more difficult - -The project's multi-package structure (`smartem_backend`, `smartem_agent`, `smartem_common`, `athena_api`) was designed -to enable modular deployment and clear separation of concerns for the high-throughput microscopy data processing -requirements. However, the API package had evolved beyond its intended scope as a simple HTTP client interface. - -Three approaches were considered for resolving the circular dependency: - -1. **Dependency inversion**: Introduce abstract interfaces to break direct import cycles whilst maintaining separate - packages -2. **Package splitting**: Further decompose packages to eliminate circular relationships through fine-grained separation -3. **Package consolidation**: Merge `smartem_api` functionality into `smartem_backend` to eliminate the circular - dependency entirely - -## Decision - -We will **consolidate the smartem_api package into smartem_backend** to eliminate the circular dependency completely. - -The consolidation will restructure the codebase as follows: -- Move `smartem_api/client.py` to `smartem_backend/api_client.py` for HTTP client functionality -- Move `smartem_api/server.py` to `smartem_backend/api_server.py` for FastAPI server implementation -- Relocate HTTP data models to `smartem_backend/model/http_*.py` alongside existing database models -- Maintain backward compatibility by creating re-export modules in the original `smartem_api/` location -- Remove `smartem_api` as a separate package from pyproject.toml configuration -- Introduce lightweight `client` dependency group for components requiring only API client functionality - -The API client will remain accessible to `smartem_agent` and external consumers through the consolidated package -structure, preserving the modularity required for distributed cryo-electron microscopy workflow processing. - -## Consequences - -**Positive:** -- Eliminates circular dependency completely, resolving architectural violation and build complexity -- Reduces operational overhead by managing one fewer package in the development and deployment pipeline -- Simplifies import structure with clear single-source responsibility for HTTP API functionality -- Maintains full backward compatibility for existing code using `smartem_api` imports -- Preserves API client reusability for `smartem_agent` and external scientific computing integrations -- Continues to support Swagger documentation generation from the consolidated API server -- Enables cleaner dependency management with optional `client` group for minimal installations - -**Negative:** -- Increases `smartem_backend` package scope, potentially affecting cognitive load for developers -- Requires migration period where both import paths coexist, creating temporary code duplication -- May impact deployment strategies that relied on separate API package for containerised microservices -- Documentation and examples require updates to reflect new package structure - -**Implementation considerations:** -- Comprehensive testing across all import paths to ensure backward compatibility during transition period -- Documentation updates for installation guides, API usage examples, and architectural diagrams -- Communication to development team about new import paths and deprecation timeline for legacy imports -- Validation of Swagger documentation generation and API client functionality post-consolidation diff --git a/webui/src/docs/explanations/decisions/0008-backend-to-agent-communication-architecture.mdx b/webui/src/docs/explanations/decisions/0008-backend-to-agent-communication-architecture.mdx deleted file mode 100644 index bf1e3d1..0000000 --- a/webui/src/docs/explanations/decisions/0008-backend-to-agent-communication-architecture.mdx +++ /dev/null @@ -1,89 +0,0 @@ -# 8. Backend-to-Agent Communication Protocol Selection - -Date: 26/08/2025 - -## Status - -Accepted - -## Context - -The SmartEM Decisions system requires a communication protocol for delivering real-time microscope control instructions from Kubernetes-hosted backend services to Windows workstation agents controlling cryo-electron microscopes. - -### Key Requirements -- **Real-time delivery**: Instructions must reach agents within seconds of generation -- **High throughput**: Support high-frequency microscopy workflows -- **Fault tolerance**: Connection failures must not result in lost instructions -- **Network compatibility**: Must work within existing network infrastructure -- **Integration**: Seamless integration with existing FastAPI and RabbitMQ architecture - -### Constraints -- Windows workstations have limited network connectivity -- Agents cannot initiate connections to backend (firewall restrictions) -- Instructions occur every 30-120 seconds per agent during data collection -- Scientific reproducibility requires reliable instruction delivery and audit trails - -## Decision - -We will implement **Server-Sent Events (SSE) for instruction streaming combined with HTTP POST acknowledgements**. - -## Alternatives Considered - -### 1. WebSockets -**Pros**: Bidirectional, low latency, excellent real-time performance -**Cons**: Complex state management, firewall compatibility issues, unnecessary complexity for 30-120 second instruction intervals, fragile in environments with routine network maintenance -**Verdict**: Unsuitable - over-engineered for microscope control timing patterns - -### 2. HTTP Long-Polling -**Pros**: Simple HTTP-based, good firewall compatibility, natural timeout handling -**Cons**: Resource intensive with blocking connections, complex timeout management, potential connection exhaustion -**Verdict**: Rejected - inefficient resource usage - -### 3. gRPC Streaming -**Pros**: Excellent performance, built-in streaming, strong typing -**Cons**: HTTP/2 proxy compatibility issues, Protocol Buffers complexity, firewall restrictions, over-engineered for infrequent instructions -**Verdict**: Rejected - unnecessary complexity - -### 4. Message Queue Pull (Direct RabbitMQ) -**Pros**: Native reliability, mature authentication, built-in failover -**Cons**: Exposes message queue to restricted networks, complex credential management, conflicts with network isolation policies -**Verdict**: Rejected - security boundary violations - -### 5. File-Based Communication -**Pros**: Simple implementation, natural persistence, no network dependencies -**Cons**: Polling overhead, poor real-time performance, file locking complexity, inadequate for sub-second requirements -**Verdict**: Rejected - insufficient performance - -## Consequences - -### Positive -- **Optimal performance**: SSE provides real-time delivery with minimal latency for microscopy workflows -- **Reliable delivery**: HTTP acknowledgements ensure instruction receipt confirmation -- **Fault tolerance**: Automatic fallback to HTTP polling during connection issues -- **Network compatibility**: HTTP/SSE protocols work within existing firewall configurations -- **Simple integration**: Leverages existing FastAPI infrastructure with minimal changes -- **Audit compliance**: HTTP-based acknowledgements provide full instruction traceability - -### Negative -- **Connection management**: SSE connections require careful lifecycle management with retry logic -- **Proxy sensitivity**: Long-lived SSE connections may be affected by corporate proxies -- **Retry complexity**: Exponential backoff logic must handle various failure scenarios - -### Polling Fallback Rejected - -Initial consideration included HTTP polling as a fallback mechanism, but this was rejected for the following reasons: - -- **Complexity without benefit**: Network issues affecting SSE typically also affect HTTP polling -- **Instruction frequency**: 30-120 second intervals make polling overhead unnecessary -- **Robust retry sufficient**: SSE with exponential backoff reconnection handles temporary failures -- **Failure correlation**: Most network problems (DNS, firewall, proxy) impact both protocols equally -- **Maintenance burden**: Dual code paths increase complexity without meaningful reliability improvement - -**Decision**: Implement robust SSE with retry logic only, without polling fallback. - -### Implementation Requirements -- FastAPI SSE endpoint with connection lifecycle management and retry logic -- HTTP acknowledgement endpoints for delivery confirmation -- Exponential backoff reconnection for SSE failures -- Integration with existing RabbitMQ event system -- Database persistence for instruction state and delivery tracking diff --git a/webui/src/docs/explanations/decisions/0009-commit-generated-route-tree.mdx b/webui/src/docs/explanations/decisions/0009-commit-generated-route-tree.mdx deleted file mode 100644 index 3315a7a..0000000 --- a/webui/src/docs/explanations/decisions/0009-commit-generated-route-tree.mdx +++ /dev/null @@ -1,110 +0,0 @@ -# ADR 0009: Commit Generated Route Tree to Version Control - -## Status - -Accepted - -## Applies To - -smartem-frontend - -## Context - -TanStack Router uses file-based routing and generates a `routeTree.gen.ts` file that contains the complete route tree for type safety. This generated file needs to be kept in sync with the route files, but there are different approaches to managing generated files: - -1. **Commit the file**: Treat it as source code and commit to version control -2. **Gitignore the file**: Add to `.gitignore` and regenerate on every build -3. **Hybrid approach**: Commit but use CI checks to verify it's up to date - -Different tools in the ecosystem handle this differently: - -- **GraphQL Codegen**: Typically gitignored, regenerated via pre-scripts -- **Prisma Client**: Typically gitignored, regenerated on install -- **TanStack Router**: TanStack officially recommends committing the file - -The key consideration is that `routeTree.gen.ts` provides TypeScript types that are needed immediately when developers clone the repository or switch branches. - -## Decision - -We will **commit `routeTree.gen.ts` to version control** and add CI verification to ensure it stays in sync with route files. - -### Implementation - -1. **Commit the generated file**: Include `src/routeTree.gen.ts` in version control -2. **Configure linter to ignore it**: Add to biome `overrides` to skip linting/formatting -3. **Add verification scripts**: - ```json - { - "router:generate": "tsr generate", - "router:check": "tsr generate && git diff --exit-code src/routeTree.gen.ts" - } - ``` -4. **CI verification**: Add `npm run router:check` to CI pipeline to fail if file is out of sync -5. **Install CLI as devDependency**: Add `@tanstack/router-cli` to ensure consistent versions - -### No Git Hooks - -We explicitly chose **not** to use git hooks (pre-commit/pre-push) because: - -- The Vite plugin already auto-generates during development -- Git hooks would create redundant generation -- Can cause confusion with "double generation" issues -- Slows down commit process unnecessarily - -## Consequences - -### Positive - -- **Type safety on clone**: Developers get TypeScript types immediately after cloning -- **Works in CI**: Type checking works without needing generation step before `tsc` -- **Branch switching**: Types are correct immediately when switching branches -- **Clear audit trail**: Changes to routes are visible in git diffs -- **Follows official guidance**: Aligned with TanStack Router's recommendation - -### Negative - -- **Merge conflicts**: Generated file can cause conflicts when multiple people modify routes - - **Mitigation**: When conflicts occur, regenerate with `npm run router:generate` rather than manually resolving -- **Repository size**: Adds ~14KB generated file to repository - - **Impact**: Negligible for most projects -- **CI dependency**: Requires CI to run verification check - - **Implementation**: Simple check: `npm run router:check` - -### Neutral - -- **Developer workflow**: No change - Vite plugin handles generation automatically during development -- **Manual sync rarely needed**: Only if someone edits routes without running dev server (rare) - -## Alternatives Considered - -### Alternative 1: Gitignore and Regenerate - -**Approach**: Add `routeTree.gen.ts` to `.gitignore` and regenerate on every build - -**Rejected because**: - -- Type checking fails immediately after clone -- CI needs extra generation step before `tsc` -- Creates "works on my machine" issues if generation differs -- Against TanStack Router's official recommendation - -### Alternative 2: Git Hooks - -**Approach**: Use pre-commit hooks to regenerate before each commit - -**Rejected because**: - -- Vite plugin already handles generation during dev -- Creates redundant/duplicate generation -- Slows down commits -- Can cause confusion about which generation is "correct" - -## References - -- [TanStack Router FAQ: Should I commit routeTree.gen.ts?](https://tanstack.com/router/latest/docs/framework/react/faq) -- [TanStack Router Discussion #1218](https://github.com/TanStack/router/discussions/1218) -- [File-Based Routing Documentation](https://tanstack.com/router/latest/docs/framework/react/guide/file-based-routing) - -## Date - -2025-01-12 diff --git a/webui/src/docs/explanations/decisions/0010-shiki-syntax-highlighting.mdx b/webui/src/docs/explanations/decisions/0010-shiki-syntax-highlighting.mdx deleted file mode 100644 index 3ed8574..0000000 --- a/webui/src/docs/explanations/decisions/0010-shiki-syntax-highlighting.mdx +++ /dev/null @@ -1,52 +0,0 @@ -# 10. Use Shiki for syntax highlighting in documentation - -Date: 12/12/2025 - -## Status - -Accepted - -## Context - -The SmartEM developer documentation contains extensive code examples across multiple languages (Python, Bash, YAML, TypeScript, SQL). Proper syntax highlighting significantly improves readability and reduces cognitive load when scanning documentation. - -Two primary options were evaluated for MDX-based syntax highlighting: - -**rehype-prism-plus** (Prism.js-based): -- Mature ecosystem with many existing themes -- Lighter bundle size and faster build times (~500ms for 400 articles) -- However, Prism v2 development has stalled (no updates in 2+ years) -- UMD module format creates integration friction with modern ESM tooling -- The original @mapbox/rehype-prism is no longer maintained as of April 2024 - -**Shiki** (via rehype-pretty-code): -- Uses the same TextMate grammar engine as VS Code -- Syntax highlighting matches exactly what developers see in their editor -- Active development with v1.0.0 released February 2024 -- Modern ESM format, integrates cleanly with Vite -- Access to VS Code's extensive theme ecosystem -- Slightly slower build times (~2s for 400 articles) and larger bundle - -## Decision - -We will use **Shiki** via the `rehype-pretty-code` plugin for syntax highlighting in MDX documentation. - -Configuration uses the `github-dark` theme to match the dark aesthetic of the developer dashboard UI. - -## Consequences - -**Positive:** -- Syntax highlighting matches VS Code exactly, reducing context-switching friction -- Access to high-quality VS Code themes (github-dark matches our UI) -- Actively maintained with modern ESM architecture -- Better accuracy for edge cases in language grammars -- Future-proof choice as Prism development has stalled - -**Negative:** -- Slightly larger bundle size compared to Prism -- Slower build times (acceptable for our documentation volume) -- Less familiar to developers who have used Prism extensively - -**Trade-offs accepted:** -- Build time increase is negligible for our ~50 documentation pages -- Bundle size increase is acceptable given the improved developer experience diff --git a/webui/src/docs/explanations/decisions/0011-remove-python-copier-template.mdx b/webui/src/docs/explanations/decisions/0011-remove-python-copier-template.mdx deleted file mode 100644 index c1ba9a5..0000000 --- a/webui/src/docs/explanations/decisions/0011-remove-python-copier-template.mdx +++ /dev/null @@ -1,55 +0,0 @@ -# 11. Remove python-copier-template - -## Status - -Accepted - -## Context - -In ADR-0002, we adopted the [python-copier-template](https://github.com/DiamondLightSource/python-copier-template) to ensure consistency in developer environments and package management. - -Since that decision, the smartem-decisions project has evolved significantly: - -1. **Project maturity**: The project has grown from a single-package PoC to a multi-package monorepo with custom requirements that diverge from standard DLS Python projects -2. **Custom tooling needs**: Our development workflow now requires tooling configurations specific to our architecture (multi-package structure, agent deployment, Kubernetes manifests, RabbitMQ integration) -3. **Template update friction**: The copier template's update mechanism became a maintenance burden rather than a benefit, as most updates were not relevant to our custom structure -4. **Duplicated documentation**: The template's contribution guidelines and developer documentation conflicted with our own evolving practices documented in smartem-devtools - -## Decision - -We have removed the python-copier-template dependency and scaffolding from smartem-decisions. - -The following were removed in commit f95b1de (2026-01-05): -- `.copier-answers.yml` configuration file -- Copier dependency from `pyproject.toml` -- Template-generated sections from `.github/CONTRIBUTING.md` - -We retain the tooling standards established by the template (pyright, ruff, pre-commit) but now manage their configuration directly. - -## Consequences - -### Positive - -- **Reduced maintenance burden**: No need to resolve conflicts when updating from the template -- **Custom workflows**: Freedom to evolve tooling and structure to match our specific needs -- **Simplified onboarding**: Developer documentation is now solely in smartem-devtools, not split between template and repo -- **Clearer ownership**: All configuration is explicitly managed by the team - -### Negative - -- **Manual updates**: We no longer automatically receive updates to best practices from the template -- **Divergence risk**: May drift from DLS Python conventions over time -- **Responsibility**: Must actively maintain tooling standards ourselves - -### Mitigations - -- Continue following DLS Python best practices where applicable -- Reference the copier template repo for inspiration when updating tooling -- Document our standards explicitly in smartem-devtools -- Maintain pre-commit hooks to enforce code quality standards - -## References - -- ADR-0002: Adopt python-copier-template (superseded by this decision) -- Removal commit: f95b1dea1479d8d845f5cfd605084c201f459020 -- DLS python-copier-template: https://github.com/DiamondLightSource/python-copier-template diff --git a/webui/src/docs/explanations/epu-data-structures.mdx b/webui/src/docs/explanations/epu-data-structures.mdx deleted file mode 100644 index f1cbde4..0000000 --- a/webui/src/docs/explanations/epu-data-structures.mdx +++ /dev/null @@ -1,76 +0,0 @@ -# EPU Output Directory Structure - -> Directory layout on the EPU machine will differ from the layout of that -> same directory synced to file storage. Layout relevant to epu data intake component -> is as it appears on the EPU machines, where the watcher runs. - -In a user visit there can be more than one `EpuSession.dm` and more than one "project dir" -as there will be one for each grid. We usually treat each `EpuSession.dm` independently, -but they all connect to the same atlas. - -``` -grid-data-dir-structure-example -├── EpuSession.dm -├── Images-Disc1 -│ ├── GridSquare_8999138 -│ │ ├── Data -│ │ │ ├── FoilHole_9015883_Data_9017347_6_20250108_154915.jpg -│ │ │ ├── FoilHole_9015883_Data_9017347_6_20250108_154915.xml -│ │ │ ├── FoilHole_9015883_Data_9017354_6_20250108_154918.jpg -│ │ │ └── FoilHole_9015883_Data_9017354_6_20250108_154918.xml -│ │ ├── FoilHoles -│ │ │ ├── FoilHole_9015889_20250108_154715.jpg -│ │ │ ├── FoilHole_9015889_20250108_154715.xml -│ │ │ ├── FoilHole_9015889_20250108_154725.jpg -│ │ │ └── FoilHole_9015889_20250108_154725.xml -│ │ ├── GridSquare_20250108_151151.jpg -│ │ └── GridSquare_20250108_151151.xml -│ └── GridSquare_8999186 -│ ├── Data -│ │ ├── FoilHole_9028219_Data_9017347_50_20250109_062621.jpg -│ │ ├── FoilHole_9028219_Data_9017347_50_20250109_062621.xml -│ │ ├── FoilHole_9028219_Data_9017354_50_20250109_062624.jpg -│ │ └── FoilHole_9028219_Data_9017354_50_20250109_062624.xml -│ ├── FoilHoles -│ │ ├── FoilHole_9028276_20250109_061712.jpg -│ │ ├── FoilHole_9028276_20250109_061712.xml -│ │ ├── FoilHole_9028276_20250109_061722.jpg -│ │ ├── FoilHole_9028276_20250109_061722.xml -│ │ └── FoilHole_9028325_20250109_063110.jpg -│ ├── GridSquare_20250108_152955.jpg -│ └── GridSquare_20250108_152955.xml -└── Metadata - ├── GridSquare_8999138.dm - └── GridSquare_8999186.dm -``` - -## EPU Directory Structure Details - -- An EPU directory consists of multiple subdirectories as well as `.xml`, `.dm` and `.jpg` files - - Any `.dm` files are actually XML files - - This directory is written to incrementally by EPU software, it does not materialise in a complete state right away -- An EPU directory will contain a file named `EpuSession.dm` at root level. Useful information contained in this file: - - Session _name_, _id_ and _start time_; - - fs path of the EPU directory; - - A reference to `Atlas.dm` file (which resides outside the EPU directory); - - _clustering_ mode and radius -- An `Atlas.dm` file contains general information about the acquisition session (which possibly duplicates info already - found in `EpuSession.dm` and, crucially, information about _atlas tiles_ - - _Atlas tiles_ contain positioning information, which is needed to map physical positions on the grid (measured in - number of turns of the actuator) to their pixel coordinates in image files -- An EPU directory will contain a subdirectory named `Metadata/` at root level - - `Metadata` directory is flat and contains a large list of files all using a naming convention - `GridSquare_.dm`, for example: `GridSquare_8999138.dm`, `GridSquare_8999186.dm` - - The `Metadata/` directory contains all grid squares not just the ones that are captured - - Some `Metadata/GridSquare_.dm` files will have a significantly larger file size than others. - Those are the ones that were of interest and so further scans took place during the acquisition session - - The rest of GridSquare files under `Metadata/` are typically around 2.8Kb in size -- An EPU directory contains at least one subdirectory named `Images-Disc1/` at root level, - and it's possible to have multiple subdirectories with a naming convention `Images-Disc`, though in most - cases there will only be one. - - An `Images-Disc/` directory will contain a number of subdirectories following a naming convention - `GridSquare_/`, corresponding to GridSquare files in `Metadata/`, but only for GridSquares of - interest where further scanning took place -- A GridSquare directory matching a glob `Images-Disc/GridSquare_/` will contain: - - a GridSquare manifest file, such as `GridSquare_20250108_152955.xml` - - **optionally** subdirectories `FoilHoles/` and `Data/`, containing FoilHole and Micrograph information, respectively diff --git a/webui/src/docs/explanations/smartem-agent-design.mdx b/webui/src/docs/explanations/smartem-agent-design.mdx deleted file mode 100644 index a32fa6a..0000000 --- a/webui/src/docs/explanations/smartem-agent-design.mdx +++ /dev/null @@ -1,868 +0,0 @@ -# SmartEM Agent: Design Specification - -**Version**: 2.0 -**Date**: 28/10/2025 -**Status**: Implemented - -## Executive Summary - -This document specifies the design for the SmartEM Agent, the real-time EPU filesystem monitoring service. The -implementation addresses non-deterministic file ordering issues that previously caused GridSquare and FoilHole -processing failures. The agent provides robust orphan handling for arbitrary entity relationship ordering, improved -bursty write handling, and enhanced code maintainability whilst preserving the proven parser implementation. - -## Background - -### Implementation Overview - -SmartEM Agent is a Windows-deployed service that monitors EPU (cryo-electron microscopy software) output directories -in real time, parsing XML metadata and synchronising entity data to the backend via REST API. The architecture -(`src/smartem_agent/`) comprises several key components: - -- **Parser** (`fs_parser.py`): XML parsing of EPU session manifests, atlas data, GridSquare metadata, FoilHole - manifests, and micrograph metadata -- **Event Classifier** (`event_classifier.py`): Classifies file events by entity type and assigns processing priority -- **Event Queue** (`event_queue.py`): Priority queue for buffering classified events during bursty writes -- **Event Processor** (`event_processor.py`): Coordinates parsing, parent checking, and orphan management -- **Orphan Manager** (`orphan_manager.py`): Manages entities awaiting parent availability with event-driven resolution -- **Error Handler** (`error_handler.py`): Categorises and handles transient vs permanent errors with retry logic -- **Watcher** (`fs_watcher.py`): Filesystem monitoring using watchdog library, orchestrating all components -- **Data Store** (`model/store.py`): In-memory entity cache with optional persistent backend synchronisation via - `PersistentDataStore`. Provides relationship tracking and natural ID lookups for deduplication - -The agent operates in two modes: -- **Dry-run mode**: In-memory data store only, no API persistence -- **Production mode**: Persistent data store with REST API synchronisation and SSE instruction streaming - -Deployment occurs as a Windows executable, with testing facilitated by fsrecorder playback simulation of EPU output -patterns. - -### Historical Problem Statement - -The original implementation exhibited critical failures when processing EPU filesystem output, manifesting primarily in -GridSquare and FoilHole entity processing during end-to-end tests. These failures stemmed from three core architectural -deficiencies that have now been addressed: - -#### 1. Insufficient Orphan Handling (Critical Priority) - -EPU filesystem output exhibits completely non-deterministic ordering characteristics: - -- **Child-before-parent sequences**: FoilHole manifests can appear before their parent GridSquare metadata files; - similarly, Micrographs can precede FoilHoles, and GridSquares can precede Atlas data. -- **Bursty file writes**: EPU buffers and dumps hundreds of files simultaneously in unpredictable sequences. -- **No ordering guarantees**: The filesystem provides no guarantees about write completion order, even for logically - related entities. - -The current orphan handling implementation (`_process_orphaned_files` method, lines 527-543 in `fs_watcher.py`) -contains several weaknesses: - -- **Grid-level orphan processing only**: Orphans are processed exclusively when new grids are detected. GridSquare and - FoilHole orphans that arrive before their parents are not handled. -- **Single retry attempt**: Orphaned files are processed once when a potential parent appears, with no mechanism for - subsequent retry if the parent-child chain remains incomplete. -- **Path-based resolution**: Relies on `get_grid_by_path` for orphan-to-grid matching, which fails for entities - requiring UUID-based parent lookups (GridSquares needing Grid UUIDs, FoilHoles needing GridSquare UUIDs). -- **No persistent orphan state**: Orphaned entities are stored only in the `orphaned_files` dictionary without timeout - tracking or systematic retry scheduling. - -This results in silent processing failures for entities arriving in unexpected orders, particularly evident in -end-to-end test failures where GridSquares and FoilHoles fail to persist to the database. - -#### 2. Inadequate Bursty Write Handling - -EPU can generate hundreds of files within seconds during data collection bursts. The current rate-limiting approach -(`log_interval` parameter, default 10 seconds) provides only event batching for logging purposes, not processing -control. Issues include: - -- **Serial processing bottleneck**: All events process sequentially in `on_any_event`, creating backlog during bursts. -- **No backpressure mechanism**: High-frequency events accumulate in `changed_files` dictionary without processing - limits. -- **Insufficient buffering strategy**: Event accumulation occurs but processing remains unbounded and - non-prioritised. - -The optimal processing model (serial versus concurrent) requires investigation. Entity interdependencies suggest serial -processing within entity hierarchies (Grid → GridSquare → FoilHole → Micrograph) may be necessary to maintain -parent-child ordering guarantees, whilst cross-hierarchy processing could potentially parallelise. - -#### 3. Code Maintainability Concerns - -The current watcher implementation exhibits technical debt that impedes modification and testing: - -- **Complex conditional nesting**: The `on_any_event` method (lines 441-515) contains deeply nested match/case - statements with interleaved file pattern matching, entity type determination, and orphan handling logic. -- **Unclear separation of concerns**: Orphan management, event routing, file classification, and entity processing - are intermingled within single methods. -- **Difficult testing**: Tight coupling between filesystem events, parsing, and persistence makes unit testing - challenging without extensive mocking. -- **Hardcoded patterns**: File pattern matching relies on regex patterns embedded throughout the code rather than - centralised configuration. - -These maintainability issues compound the difficulty of implementing robust orphan handling, as the current structure -resists modification without introducing regression risks. - -## Design Goals and Principles - -The SmartEM Agent v2 design adheres to the following principles: - -### Primary Goals - -1. **Robust orphan handling**: Support arbitrary entity arrival ordering without assumptions about parent-child - sequence. No entity should fail to process due to parent unavailability at initial observation time. -2. **Bursty write resilience**: Handle high-frequency file creation bursts (hundreds of files within seconds) without - processing failures, dropped events, or unbound resource consumption. -3. **Maintainability**: Produce clear, testable code with well-separated concerns, enabling future modifications - without architectural rework. -4. **Reliability**: Ensure eventual consistency between filesystem state and datastore state, with comprehensive error - recovery mechanisms. - -### Design Principles - -- **No ordering assumptions**: The system must not assume any specific file arrival sequence. Parents may arrive - after children; siblings may arrive in arbitrary order. -- **Idempotency**: Entity processing operations must be safely repeatable. Re-processing a file should produce - identical datastore state. -- **Separation of concerns**: File watching, event classification, parsing, orphan management, and persistence should - occupy distinct, loosely coupled components. -- **Testability**: Components should support unit testing without filesystem dependencies or extensive mocking. -- **Incremental processing**: Large batches should process incrementally to prevent resource exhaustion and enable - progress monitoring. -- **Observability**: Processing state, orphan status, and retry attempts should be visible through structured logging - and metrics. - -### Constraints - -- **Dual-ID system**: Every entity maintains both a synthetic UUID (v4, agent-generated, database primary key) and a - natural ID (EPU-generated, e.g. GridSquare "1", FoilHole "2-3"). UUID generation occurs immediately when entity data - is first created. Natural IDs are essential for parent-child relationship resolution and orphan matching. This - dual-ID architecture is critical for the entire system and must be preserved. -- **Dual data store model**: Maintain both in-memory and persistent stores. The in-memory store serves as cache for - relationship lookups, development convenience, and architectural modularity. -- **Windows deployment compatibility**: Solution must build to Windows executable for Win10/11 deployment. -- **Backwards compatibility preference**: Avoid breaking changes where practical, though not a strict requirement as - the system is not yet in production. -- **Parser preservation**: `fs_parser.py` works reliably and should remain largely unchanged. - -## Proposed Architecture - -### Dual-ID System: Foundation for Orphan Resolution - -Understanding the dual-ID system is critical for comprehending the orphan handling strategy. - -#### Identity Management - -Every entity in the system maintains two identifiers: - -1. **Synthetic UUID** (Primary Key): - - Generated agent-side using UUID v4 - - Created immediately when entity data structure is instantiated - - Database primary key and system-wide identifier - - Used for all parent-child relationships in Data Store and backend database - - Never appears in EPU files - -2. **Natural ID** (EPU Identifier): - - Originates from EPU software (e.g., GridSquare "1", FoilHole "2-3", Micrograph "2-3-1") - - Found in EPU XML files and directory names - - Used for entity lookup during parent relationship resolution - - Enables deduplication (same natural ID = same entity, reuse UUID) - -#### Parent Relationship Resolution - -When parsing a FoilHole file, the process works as follows: - -1. **Parser extracts natural IDs**: - - FoilHole natural ID: "2-3" - - Parent GridSquare natural ID: "2" (extracted from file path or XML content) - -2. **UUID lookup for parent**: - - Call `datastore.find_gridsquare_by_natural_id("2")` - - If found: retrieve parent's UUID → set `foilhole.gridsquare_uuid = parent.uuid` - - If not found: **orphan detected** (parent doesn't exist yet) - -3. **UUID management for entity**: - - Call `datastore.find_foilhole_by_natural_id("2-3")` - - If found (update scenario): reuse existing UUID - - If not found (new entity): UUID already generated during entity creation - -4. **Persistence**: - - With parent UUID available: persist FoilHole with complete relationship data - - Without parent UUID: register as orphan awaiting parent appearance - -#### Orphan Detection Mechanism - -An entity becomes orphaned when: -- Parser successfully extracts entity data from EPU file -- Natural ID lookup for required parent returns None -- Persistence attempt fails parent existence check - -Example orphan scenarios: -- FoilHole "2-3" parsed but GridSquare "2" doesn't exist yet -- Micrograph "2-3-1" parsed but FoilHole "2-3" doesn't exist yet -- GridSquare "5" parsed but parent Grid hasn't been created yet (rare, handled by current implementation) - -#### Orphan Resolution Strategy: Event-Driven Matching - -**Key insight**: Orphan resolution is event-driven, not periodic retry-based. - -When a parent entity appears: -1. **Process parent normally**: Parse file, generate/retrieve UUID, persist to Data Store -2. **Check orphan registry**: Query for orphans requiring this parent's natural ID -3. **Resolve matching orphans**: For each orphan: - - Set orphan's parent UUID (now available) - - Persist orphan to Data Store - - Remove from orphan registry - -Example flow - FoilHole arrives before GridSquare: -1. FoilHole "2-3" file appears -2. Parser extracts: natural ID "2-3", parent GridSquare natural ID "2" -3. Lookup GridSquare "2": **not found** → orphan -4. Register orphan: `{natural_id: "2-3", parent_natural_id: "2", entity_data: }` -5. *(later)* GridSquare "2" file appears -6. Parse and persist GridSquare "2" (gets UUID) -7. **Check orphans**: "Are any orphans waiting for GridSquare '2'?" -8. Find FoilHole "2-3" orphan -9. Set `foilhole.gridsquare_uuid = gridsquare.uuid` -10. Persist FoilHole "2-3" -11. Remove from orphan registry - -This event-driven approach provides immediate orphan resolution with zero polling overhead. - -### High-Level Component Structure - -The v2 architecture introduces four new components whilst preserving the proven Parser and Data Store implementations: - -- **Filesystem Watcher** → **Event Classifier** → **Event Queue** → **Event Processor** → **Parser** / **Orphan Manager** - → **Data Store** → **Backend REST API** - -New components (Event Classifier, Event Queue, Event Processor, Orphan Manager) handle event buffering, classification, -and orphan resolution. Existing components (Parser, Data Store) remain unchanged. - -### Component Descriptions - -#### Filesystem Watcher (Unchanged Externally) - -Continues using watchdog's `FileSystemEventHandler` for monitoring file creation and modification events. Interface -remains compatible with existing deployment. Internal processing delegation changes to route events through new -components. - -**Responsibilities**: -- Monitor watch directory for file creation/modification events -- Filter events by configured file patterns -- Delegate event processing to Event Classifier - -**No changes** to watchdog integration or SSE client management. - -#### Event Classifier (New Component) - -Examines file paths and classifies events by entity type and processing priority. - -**Responsibilities**: -- Apply regex patterns to determine entity type (Grid, Atlas, GridSquare, FoilHole, Micrograph) -- Extract natural IDs from file paths (GridSquare ID, FoilHole ID, etc.) -- Assign processing priority based on entity hierarchy (Grid > Atlas > GridSquare > FoilHole > Micrograph) -- Enqueue classified events to Event Queue - -**Interface**: -```python -@dataclass -class ClassifiedEvent: - entity_type: EntityType # Enum: GRID, ATLAS, GRIDSQUARE, FOILHOLE, MICROGRAPH - file_path: Path - natural_id: str | None # Extracted ID where applicable - priority: int # Lower number = higher priority - timestamp: float - event_type: str # 'created' or 'modified' -``` - -**Processing Logic**: -- Match file path against known patterns from `EpuParser` -- Extract identifiers using existing regex patterns -- Priority assignment: Grid=0, Atlas=1, GridSquare=2, FoilHole=3, Micrograph=4 -- Unknown files are logged and discarded - -#### Event Queue (New Component) - -Buffered queue for classified events with priority ordering and batch processing support. - -**Responsibilities**: -- Buffer incoming classified events during bursty writes -- Maintain priority ordering (higher-priority entities process first) -- Provide batch retrieval for processing -- Support backpressure via size limits -- Track queue depth metrics for monitoring - -**Interface**: -```python -class EventQueue: - def enqueue(self, event: ClassifiedEvent) -> None: ... - def dequeue_batch(self, max_size: int = 50) -> list[ClassifiedEvent]: ... - def size(self) -> int: ... - def clear(self) -> None: ... -``` - -**Configuration**: -- Maximum queue size: 1000 events (configurable) -- Batch size: 50 events per processing cycle (configurable) -- Priority queue implementation using `heapq` or `queue.PriorityQueue` - -#### Event Processor (New Component - Core Orchestrator) - -Coordinates event processing, delegating to Parser and Orphan Manager based on parent availability. - -**Responsibilities**: -- Retrieve event batches from Event Queue -- Delegate parsing to `EpuParser` -- Check parent entity availability in Data Store -- Route successful parses to Data Store for persistence -- Route orphaned entities (missing parents) to Orphan Manager -- Handle processing errors with logging and optional retry - -**Processing Flow**: -``` -For each event in batch: - 1. Parse file using EpuParser → entity data - 2. Check parent exists in Data Store: - - If parent exists: persist entity via Data Store - - If parent missing: register with Orphan Manager - 3. Trigger Orphan Manager to retry orphans (parents may now exist) - 4. Log processing outcome (success/orphan/error) -``` - -**Interface**: -```python -class EventProcessor: - def __init__(self, parser: EpuParser, datastore: DataStore, orphan_mgr: OrphanManager): ... - def process_batch(self, events: list[ClassifiedEvent]) -> ProcessingStats: ... -``` - -#### Orphan Manager (New Component - Critical for Reliability) - -Manages entities awaiting parent availability with event-driven resolution. - -**Responsibilities**: -- Store orphaned entities indexed by required parent natural ID -- Resolve orphans immediately when their parent entity appears -- Detect and log permanently orphaned entities (timeout-based) -- Provide orphan status visibility for debugging - -**Data Structures**: -```python -@dataclass -class OrphanedEntity: - entity_type: EntityType - entity_data: GridSquareData | FoilHoleData | MicrographData | ... - required_parent_type: EntityType - required_parent_natural_id: str # Natural ID of parent (e.g., "2" for GridSquare) - file_path: Path - first_seen: float # Timestamp for timeout detection -``` - -**Orphan Storage**: -```python -# Index orphans by parent natural ID for fast lookup -orphans_by_parent: dict[tuple[EntityType, str], list[OrphanedEntity]] -# Key: (parent_entity_type, parent_natural_id) -# Value: List of orphans waiting for this parent -# Example: (GRIDSQUARE, "2") → [FoilHole_2-3, FoilHole_2-5, Micrograph_2-1-1] -``` - -**Event-Driven Resolution Strategy**: - -When Event Processor successfully persists a parent entity (e.g., GridSquare "2"): -1. **Trigger orphan check**: Call `orphan_manager.resolve_orphans_for(GRIDSQUARE, "2")` -2. **Lookup matching orphans**: Retrieve all orphans waiting for `(GRIDSQUARE, "2")` -3. **Resolve each orphan**: - - Lookup parent entity in Data Store by natural ID (now guaranteed to exist) - - Extract parent's UUID - - Set orphan's parent UUID field (e.g., `foilhole.gridsquare_uuid = gridsquare.uuid`) - - Persist orphan to Data Store - - Remove from orphan registry - - Log successful resolution -4. **Cascading resolution**: Resolving an orphan may enable resolution of its children - - Example: Resolving GridSquare "2" may enable FoilHole "2-3", which may enable Micrograph "2-3-1" - - Event Processor recursively checks orphans after each successful persistence - -**Timeout Detection** (safeguard for genuinely missing files): -- Periodic background task (every 60 seconds) checks orphan timestamps -- Orphans exceeding timeout threshold (e.g., 5 minutes) logged as permanently orphaned -- Enables operator intervention for corrupted or incomplete EPU sessions - -**Interface**: -```python -class OrphanManager: - def register_orphan(self, entity_data, required_parent_type, required_parent_natural_id, file_path): ... - def resolve_orphans_for(self, parent_type: EntityType, parent_natural_id: str) -> int: ... - def check_timeouts(self, max_age_seconds: float = 300) -> list[OrphanedEntity]: ... - def get_orphan_stats(self) -> dict[EntityType, int]: ... -``` - -#### Data Store (Minimal Changes) - -Existing `InMemoryDataStore` and `PersistentDataStore` implementations remain largely unchanged. Minor enhancements: - -**Additions**: -- `has_parent(entity_type, parent_id) -> bool`: Check parent existence without retrieving full entity -- Enhanced logging for API retry failures (already present but may need refinement) - -**No changes** to UUID generation, relationship tracking, or API synchronisation logic. - -#### Parser (No Changes) - -`EpuParser` remains unchanged. All existing parsing methods continue functioning as designed. - -### Data Flow - -The event processing flow proceeds through distinct stages: - -1. **Event Detection**: Filesystem Watcher detects file creation/modification events -2. **Classification**: Event Classifier determines entity type and assigns priority -3. **Buffering**: Event Queue stores classified events with priority ordering -4. **Batch Processing** (periodic, e.g. every 100ms): - - Event Processor dequeues batch (up to 50 events) - - For each event: - - Parser extracts entity data from file - - Event Processor checks parent exists in Data Store - - If parent exists: persist entity to Data Store → Backend API - - If parent missing: register with Orphan Manager -5. **Orphan Resolution** (event-driven): After successfully persisting each entity: - - Event Processor calls `orphan_manager.resolve_orphans_for(entity_type, natural_id)` - - Orphan Manager finds all orphans waiting for this parent - - Resolves each orphan by completing parent relationship and persisting - - Cascading: Resolving an orphan triggers check for its own children orphans - -**Timeout detection**: Periodic background task (every 60s) identifies orphans exceeding age threshold (5 minutes) for -operator intervention. - -### Orphan Resolution Strategy (Detailed) - -The orphan resolution strategy is the core innovation in v2, addressing the primary failure mode of v1. This section -provides detailed rationale and implementation specifics. - -#### Why Orphans Occur - -Orphans arise from three filesystem characteristics: - -1. **Non-atomic writes**: EPU writes files individually, not as atomic transactions. A GridSquare's FoilHole manifests - may complete writing before the GridSquare's own metadata file finishes writing. -2. **Buffered writes**: Operating system write buffering and EPU's internal buffering can reorder file completion - relative to logical creation order. -3. **Multi-directory writes**: EPU writes to multiple directories (`Metadata/`, `Images-Disc*/`, etc.) concurrently, - with no synchronisation guarantees across directory boundaries. - -#### Orphan Detection - -An entity is orphaned if its required parent is not present in the Data Store at processing time. Parent requirements: - -| Entity Type | Required Parent Type | Parent Identifier | -|---------------|----------------------|------------------------------| -| GridSquare | Grid | Grid UUID (from path lookup) | -| FoilHole | GridSquare | GridSquare UUID (from ID) | -| Micrograph | FoilHole | FoilHole UUID (from ID) | -| Atlas | Grid | Grid UUID (from path lookup) | -| AtlasTile | Atlas | Atlas UUID | - -Grid entities never orphan (they are root entities). Atlas and AtlasTile orphaning is theoretically possible but -extremely rare in practice (Atlas.dm typically completes before GridSquare files). - -#### Event-Driven Resolution Rationale - -The v2 orphan resolution strategy is event-driven, not retry-based. This fundamental design choice provides several -advantages: - -**Zero latency resolution**: When a parent entity appears and is persisted, orphan resolution occurs immediately in the -same processing cycle. No polling intervals, no exponential backoff delays. - -**No wasted cycles**: Resolution attempts only occur when new parents arrive. Traditional periodic retry systems waste -CPU checking orphans when no new parents exist. - -**Cascading resolution**: Resolving an orphan (e.g., GridSquare "2") immediately triggers resolution checks for its -children (FoilHoles waiting for GridSquare "2"), enabling multi-level dependency chains to resolve in a single event -processing cycle. - -**Deterministic behavior**: Resolution timing depends solely on parent arrival, not on retry scheduling parameters. -This simplifies testing and debugging. - -#### Resolution Mechanism - -When Event Processor successfully persists any entity: - -1. **Extract entity identity**: Determine entity type and natural ID (e.g., GRIDSQUARE, "2") -2. **Trigger resolution**: Call `orphan_manager.resolve_orphans_for(GRIDSQUARE, "2")` -3. **Orphan Manager executes**: - - Lookup orphans indexed by `(GRIDSQUARE, "2")` - - For each matching orphan (e.g., FoilHole "2-3"): - - Retrieve parent entity from Data Store by natural ID - - Extract parent's UUID - - Complete orphan's parent relationship (set `foilhole.gridsquare_uuid`) - - Persist orphan to Data Store - - Remove from orphan registry - - **Recursively trigger resolution** for this orphan's natural ID (in case it has children) - -This cascading resolution enables arbitrarily deep dependency chains to resolve in a single processing cycle. - -#### Orphan Storage - -Orphans persist in memory within `OrphanManager` using a dictionary indexed by required parent: - -```python -orphans_by_parent: dict[tuple[EntityType, str], list[OrphanedEntity]] -# Key: (required_parent_type, required_parent_natural_id) -# Value: List of orphans waiting for this parent -# Example: (GRIDSQUARE, "2") → [FoilHole "2-3", FoilHole "2-5", Micrograph "2-1-1"] -``` - -This indexing structure enables: -- O(1) lookup of all orphans waiting for a specific parent -- Efficient bulk resolution when parent appears -- Natural support for one-to-many parent-child relationships - -Orphans do not persist across agent restarts. Upon restart, filesystem re-scanning will re-discover orphaned files, -re-triggering orphan registration. This design trades restart recovery complexity for implementation simplicity, -acceptable given infrequent agent restarts during active data collection. - -#### Timeout Detection - -Event-driven resolution handles >99% of orphan scenarios. For genuinely missing parent files (corrupted EPU sessions, -incomplete transfers): - -- Periodic background task (every 60 seconds) scans all orphans -- Orphans exceeding age threshold (default: 5 minutes) logged as permanently orphaned -- Logs include: entity type, natural ID, required parent, first seen timestamp, file path -- Enables operator intervention for data recovery or session debugging - -## Key Design Decisions - -### Decision 1: Serial vs Concurrent Event Processing - -**Question**: Should events process serially (one at a time) or concurrently (parallel processing)? - -**Analysis**: - -**Serial Processing**: -- **Pros**: Maintains strict entity hierarchy ordering; simplifies parent dependency checking; avoids race conditions - in Data Store access; straightforward implementation -- **Cons**: Potentially slower throughput during extreme bursts (hundreds of files); single-threaded processing - bottleneck - -**Concurrent Processing**: -- **Pros**: Higher throughput potential; better CPU utilisation on multi-core systems; reduced processing latency - during bursts -- **Cons**: Complex parent dependency coordination; requires Data Store locking/synchronisation; race conditions - between orphan registration and parent creation; significantly increased implementation complexity - -**Decision**: **Serial processing** within the Event Processor. - -**Rationale**: -1. **Entity interdependencies**: Parent entities must exist before children can persist. Concurrent processing risks - child processing completing before parent processing, artificially creating orphans. -2. **Throughput sufficiency**: Parsing and persisting a single entity takes `<10ms` in typical cases. Serial processing - sustains ~100 entities/second, far exceeding typical EPU output rates (10-20 entities/second during active - collection). -3. **Simplicity**: Serial processing eliminates concurrency bugs, Data Store locking requirements, and orphan - registration race conditions. -4. **Burst handling**: Event Queue buffering and batch processing (50 events/batch) provide sufficient burst - absorption. Processing 50 events serially takes ~500ms, acceptable latency. - -**Future consideration**: If profiling reveals serial processing bottlenecks, investigate hierarchical parallelism -(concurrent processing across independent Grid hierarchies, serial within each hierarchy). - -### Decision 2: Orphan Storage Location - -**Question**: Should orphans store in-memory, in persistent Data Store, or in separate storage? - -**Decision**: **In-memory storage** within `OrphanManager`. - -**Rationale**: -1. **Lifecycle**: Orphans are transient processing state, not domain entities. They should not pollute the Data Store - entity collections. -2. **Persistence unnecessary**: Orphans resolve within seconds to minutes. Persistence across agent restarts provides - minimal value given filesystem re-scanning automatically re-discovers orphaned files. -3. **Simplicity**: In-memory storage avoids Data Store schema changes, API synchronisation complexity, and database - migrations. -4. **Performance**: In-memory lookups are faster than database queries, beneficial for high-frequency retry attempts. - -**Trade-off**: Orphan state loss on agent restart. Acceptable given infrequent restarts during active data collection -and automatic re-discovery upon restart. - -### Decision 3: Event Queue Implementation - -**Question**: What data structure and size limits should the Event Queue use? - -**Decision**: **Priority queue** with **1000 event maximum size** and **50 event batch size**. - -**Rationale**: -1. **Priority queue**: Ensures higher-priority entities (Grids, Atlases) process before lower-priority entities - (Micrographs), reducing orphan occurrence by processing parents first when possible. -2. **1000 event limit**: Accommodates extreme bursts (EPU rarely exceeds 500 files/burst) whilst preventing unbounded - memory growth. Exceeding limit triggers warning logging and oldest event eviction. -3. **50 event batch size**: Balances processing latency (500ms at 10ms/entity) with orphan resolution responsiveness. - Smaller batches increase orphan resolution frequency but reduce throughput; larger batches improve throughput but - delay orphan resolution (orphans resolve when their parent is persisted). - -**Configuration**: All parameters (queue size, batch size, priorities) should be configurable via environment -variables or configuration file for deployment-specific tuning. - -### Decision 4: Event-Driven vs Periodic Retry Orphan Resolution - -**Question**: Should orphan resolution use event-driven triggers or periodic retry with backoff? - -**Decision**: **Event-driven resolution** triggered when parent entities are persisted. - -**Rationale**: -1. **Zero latency**: Orphans resolve immediately when their parent appears, not on next retry cycle. Typical resolution - time: `<1ms` after parent persistence. -2. **No wasted cycles**: Resolution attempts only occur when new parents arrive. Periodic retry wastes CPU checking - orphans during quiet periods. -3. **Cascading resolution**: Resolving an orphan (e.g., GridSquare) immediately enables resolution of its children - (FoilHoles), allowing multi-level dependency chains to resolve in a single processing cycle. -4. **Deterministic testing**: Resolution timing depends solely on parent arrival order, not on retry intervals or - backoff parameters. Simplifies test case design and debugging. -5. **Simpler implementation**: No retry scheduling, no exponential backoff calculations, no retry count tracking. - -**Trade-off**: Requires indexing orphans by required parent natural ID rather than simple list storage. The O(1) lookup -performance benefit outweighs the marginal increase in storage complexity. - -**Timeout detection**: Periodic background task (60s interval) identifies orphans exceeding age threshold (5 minutes) -for genuinely missing parent files. Timeouts generate warning logs but do NOT evict orphans from memory, supporting -long-running production sessions (48+ hours) where parent files may arrive after extended delays. Memory footprint for -orphans is negligible (~750 bytes per orphan, ~1.9 MB for 2,500 orphans in extreme scenarios). - -## Implementation Strategy - -### Phased Development Approach - -The implementation was completed in phases, initially developed in `src/smartem_agent2/` alongside the original -implementation to enable iterative development. After thorough testing and validation, the new implementation replaced -the original in `src/smartem_agent/`. - -#### Phase 1: Foundation Components (Week 1-2) - -**Deliverables**: -- `event_classifier.py`: Event classification with entity type detection and priority assignment -- `event_queue.py`: Priority queue implementation with configurable size limits -- `orphan_manager.py`: Orphan storage and basic retry scheduling (without exponential backoff initially) -- Unit tests for all three components using synthetic events (no filesystem dependencies) - -**Validation**: -- Unit test coverage >90% for new components -- Event classification correctly identifies all EPU entity types from sample file paths -- Event queue maintains priority ordering under random insertion -- Orphan manager stores and retrieves orphans by entity type and natural ID - -#### Phase 2: Event Processor and Integration (Week 3-4) - -**Deliverables**: -- `event_processor.py`: Batch processing orchestration with Parser and Data Store integration -- Integration of Event Classifier, Event Queue, Event Processor, and Orphan Manager -- Adaptation of `fs_watcher.py` to delegate events to Event Classifier (minimal changes to existing class) -- Integration tests using fsrecorder playback with small datasets (10-20 GridSquares) - -**Validation**: -- Event Processor correctly processes batches with known file sets -- Orphan Manager successfully resolves GridSquare and FoilHole orphans in test datasets -- Integration tests pass with zero entity loss and zero orphan leakage -- Performance benchmarking: process 100 entities within 2 seconds (serial processing) - -#### Phase 3: Transient Error Retry and Enhanced Observability (Week 5-6) - -**Context**: Production environment differs significantly from dev/test: -- Production: Windows binary, live microscope, real-time file writes over 4-48 hours -- Dev/Test: Linux interpreted Python, fsrecorder simulation with accelerated timing (100-1000x speed) -- Primary goal: Address dev/test brittleness from accelerated playback whilst improving production reliability - -**Deliverables**: -- Exponential backoff retry for transient parser errors (corrupted XML during write, file locks) -- Exponential backoff retry for transient API connection failures (network timeouts, service restarts) -- Agent log streaming to backend (via SSE or dedicated endpoint) for centralized monitoring -- Enhanced error categorization (transient vs permanent failures) -- Structured metrics: processing latency percentiles (p50, p95, p99), retry distributions, error counts -- Enhanced SSE instructions: detailed statistics, orphan inspection, error reporting -- End-to-end tests using fsrecorder playback with multiple timing modes (--fast, --dev-mode, --exact) -- Full-scale E2E test with bi37708-42 dataset (8,389 events) - -**Validation**: -- Exponential backoff correctly retries transient errors without blocking event processing -- Permanent errors (corrupt files, missing parents after timeout) logged appropriately without retry -- Orphan timeout detection generates warnings but does NOT evict orphans (supports 48h+ sessions) -- End-to-end tests pass with all fsrecorder timing modes (100x fast, 1000x dev-mode, 1x exact) -- Agent logs successfully stream to backend for centralized monitoring -- Processing metrics visible in logs and SSE instruction responses - -#### Phase 4: Performance Optimisation and Production Hardening (Week 7-8) - -**Deliverables**: -- Performance profiling and optimisation based on fsrecorder playback results -- Configuration externalisation (queue sizes, retry parameters, batch sizes) -- Comprehensive error recovery testing (API failures, parser errors, corrupted files) -- Documentation updates (README, API documentation, deployment guides) -- Windows executable build and deployment testing - -**Validation**: -- Performance targets met: process 200 entities/second (serial), orphan resolution within 5 seconds for 95th - percentile -- Configuration changes apply without code modification -- Error recovery scenarios handled gracefully without data loss -- Windows executable deploys and operates correctly on Win10/11 test systems - -### Testing Strategy - -Testing employs a multi-layered approach progressing from unit tests to full end-to-end validation: - -#### Unit Tests - -Each component (`EventClassifier`, `EventQueue`, `OrphanManager`, `EventProcessor`) receives dedicated unit test -coverage with mocked dependencies. Focus areas: - -- **Event Classifier**: File path pattern matching, entity type detection, priority assignment, natural ID extraction -- **Event Queue**: Priority ordering, batch retrieval, size limit enforcement, edge cases (empty queue, single-element - batches) -- **Orphan Manager**: Orphan registration, event-driven resolution, timeout detection (warning-only, no eviction), - orphan statistics by type and age -- **Event Processor**: Batch processing, parser delegation, parent checking, orphan registration, Data Store - interaction - -**Target**: >90% code coverage across new components. - -#### Integration Tests - -Integration tests use fsrecorder playback to simulate EPU filesystem output patterns with controlled file ordering -and multiple timing modes: - -**Timing modes** (fsrecorder replay options): -- `--fast` (100x speed, 1s max delays): Balanced mode for realistic testing, DEFAULT -- `--dev-mode` (1000x speed, burst): Maximum acceleration for rapid iteration and stress testing -- `--exact` (1x speed): Preserve original timing for debugging timing-dependent issues - -**File ordering scenarios**: -- **Normal ordering**: Parents before children (Grid → Atlas → GridSquare → FoilHole → Micrograph) -- **Reverse ordering**: Children before parents (Micrograph before FoilHole, FoilHole before GridSquare, etc.) -- **Mixed ordering**: Random interleaving of parent and child files -- **Bursty writes**: Hundreds of files appearing simultaneously -- **Incomplete hierarchies**: Missing parent files (Atlas absent, GridSquare metadata missing) - -**Validation**: -- Zero entity loss across all ordering scenarios and timing modes -- Orphan resolution success rate >99% for complete hierarchies -- Orphaned entities with missing parents log timeout warnings (no eviction) -- Processing latency `<5 seconds` for 95th percentile entities (fast mode) - -#### End-to-End Tests - -End-to-end tests use full-scale fsrecorder datasets (100+ GridSquares, 1000+ FoilHoles) with realistic EPU output -patterns. Primary test dataset: bi37708-42 (8,389 events). Tests validate: - -- Complete entity persistence to backend database -- SSE instruction handling during active data collection -- Agent restart recovery (orphan re-discovery) -- Transient error retry (parser errors with file locks, API connection failures) -- Agent log streaming to backend for centralized monitoring -- Windows executable deployment and operation (production environment) - -**Target**: All existing end-to-end tests pass with v2 implementation, plus additional tests for: -- Orphan timeout warning scenarios (no eviction) -- Transient error recovery with exponential backoff -- Long-running sessions (48+ hours simulated via extended playback) -- All three fsrecorder timing modes (fast, dev-mode, exact) - -## Open Questions and Future Considerations - -### Open Questions for Investigation - -#### 1. Optimal Batch Size - -**Question**: Is 50 events/batch optimal for throughput and orphan resolution trade-off? - -**Investigation Plan**: Benchmark processing latency and orphan resolution time across batch sizes: 10, 25, 50, 100, -200 events/batch using fsrecorder playback with large datasets. Measure: -- Total processing time for complete dataset -- 95th percentile orphan resolution time -- Memory consumption during processing - -**Decision Criteria**: Select batch size minimising orphan resolution time whilst maintaining total processing time -`<10%` above minimum observed (to avoid sacrificing throughput for marginal resolution improvements). - -#### 2. Persistent Orphan Storage - -**Question**: Should orphans persist across agent restarts for long-running orphan scenarios? - -**Current Decision**: No persistent storage (in-memory only). - -**Reconsideration Triggers**: -- If operational data reveals orphans persisting beyond 5 minutes (indicating parent files genuinely missing rather - than delayed) -- If agent restarts during active data collection prove common in production -- If filesystem re-scanning performance becomes problematic for large datasets - -**Implementation Path**: If persistence becomes necessary, store orphans in SQLite database co-located with agent, -avoiding backend API dependency for transient processing state. - -#### 3. Hierarchical Parallelism - -**Question**: Could concurrent processing across independent Grid hierarchies improve throughput without introducing -parent-child race conditions? - -**Current Decision**: Serial processing (see Decision 1). - -**Future Investigation**: If performance profiling reveals serial processing bottlenecks (unlikely based on -preliminary analysis), investigate: -- Processing batches grouped by Grid UUID (parallel across Grids, serial within Grid) -- Thread pool or async/await implementation for Grid-level concurrency -- Lock-free Data Store design for concurrent Grid insertion - -**Decision Criteria**: Pursue only if serial processing demonstrably fails to meet throughput requirements (>200 -entities/second sustained). - -### Future Enhancements (Out of Scope for v2) - -The following features are explicitly deferred to future iterations: - -#### Athena API Integration Modes - -**Description**: Three-mode operation for microscope control: -- **Dry-run mode**: Data ingestion only, no microscope control -- **Semi-automatic mode**: Athena API generates recommendations, operator approves -- **Fully automatic mode**: Athena API directly controls microscope via SSE instructions - -**Deferral Rationale**: Core orphan handling and event processing must stabilise before introducing complex microscope -control workflows. Athena API integration builds upon reliable data ingestion foundation. - -#### Data Intake Only Mode - -**Description**: Disable SSE instruction streaming, operate purely as data ingestion service for offline analysis. - -**Deferral Rationale**: Current SSE implementation is stable and non-invasive. Disabling SSE provides minimal value -until specific use cases emerge requiring intake-only operation. - -#### Session Completion Detection - -**Description**: Automatically detect EPU session completion (no new files for threshold duration, session manifest -indicates completion, etc.) and trigger finalisation workflows. - -**Deferral Rationale**: Session completion detection requires understanding EPU's session lifecycle signals, which -remain partially undocumented. Reliable detection mechanisms require field observation across multiple sessions. - -#### Multi-Agent Coordination - -**Description**: Support multiple agents monitoring different microscopes within single backend instance, with -session-based isolation and cross-agent coordination. - -**Deferral Rationale**: Multi-agent scenarios are not yet tested in production. Backend session isolation mechanisms -require validation before agent-side coordination features become relevant. - -## Conclusion - -The SmartEM Agent design addresses critical orphan handling deficiencies through systematic component separation, -intelligent error handling with retry logic, and robust event buffering. The implementation provides reliable -entity processing across all tested scenarios. - -Key innovations implemented: -- **Orphan Manager** with event-driven resolution and timeout warnings -- **Event Queue** priority-based buffering for bursty write handling -- **Event Processor** orchestration with clear separation of concerns -- **Error Handler** with categorisation and exponential backoff retry -- **Preservation of proven components** (Parser, Data Store) - -The implementation supports existing deployment patterns (Windows executable, fsrecorder testing), and provides clear -extension points for future features (Athena API integration, session completion detection, multi-agent coordination). - -The system has been validated with comprehensive testing demonstrating zero-entity-loss reliability across all test -scenarios including various file ordering patterns and timing modes. diff --git a/webui/src/docs/explanations/technical-notes.mdx b/webui/src/docs/explanations/technical-notes.mdx deleted file mode 100644 index a882837..0000000 --- a/webui/src/docs/explanations/technical-notes.mdx +++ /dev/null @@ -1,64 +0,0 @@ -# Technical Notes - -## Cryo-EM Data Acquisition Scale - -Understanding the scale and volume of cryo-electron microscopy data acquisition is crucial for system design and performance considerations: - -### Micrograph Statistics -- **Micrographs per foil hole**: Typically 4-10 images per foil hole -- **Grid-level acquisition**: 10,000-50,000 micrographs per complete grid session -- **Particle density**: Approximately 300 particles identified per micrograph -- **Selection efficiency**: Roughly 50% of identified particles are selected for further processing - -These statistics inform the design requirements for data processing pipelines and storage allocation strategies. - -## System Architecture Considerations - -### Decision-Making Framework -The SmartEM system implements a modular decision-making architecture designed for flexibility and extensibility: - -- **Modular design**: Decision-making components are decoupled from data acquisition systems -- **Pluggable authorities**: Different decision-making algorithms can be easily substituted -- **Future extensibility**: Architecture supports integration of additional decision-making systems - -### API Integration -Communication between SmartEM components and external cryo-EM control systems occurs through well-defined API interfaces, ensuring compatibility with various microscope control software packages. - -## Data Management and Storage - -### ISPyB Integration -The project integrates with the [ISPyB database schema](https://github.com/DiamondLightSource/ispyb-database), which provides comprehensive metadata storage for: - -- Experimental run information -- Session metadata and parameters -- Image counts and acquisition statistics -- Sample type and classification data - -### File System Organisation -Data acquisition produces structured file system layouts that facilitate automated processing and quality assessment workflows. - -## Processing Pipeline Components - -### Particle Processing Workflow -The automated processing pipeline comprises several specialised services: - -#### Particle Picking Service -- **Input**: JSON message via RabbitMQ containing image path and processing parameters -- **Processing**: Automated particle identification and coordinate extraction -- **Output**: List of particle coordinates for each processed micrograph -- **Implementation**: [CryOLO service](https://github.com/DiamondLightSource/cryoem-services/blob/main/src/cryoemservices/services/cryolo.py) - -#### Particle Selection Service -- **Function**: Quality-based filtering and selection of identified particles -- **Integration**: Seamless integration with particle picking results -- **Implementation**: [Selection service](https://github.com/DiamondLightSource/cryoem-services/blob/main/src/cryoemservices/services/select_particles.py) - -## Scientific Context and References - -The SmartEM system builds upon established methodologies and practices in automated cryo-electron microscopy: - -### Key Publications -- [Structural Biology Methods](https://www.ncbi.nlm.nih.gov/pmc/articles/PMC10910546/) - Comprehensive overview of automated cryo-EM data collection strategies -- [Advanced Automation Techniques](https://www.biorxiv.org/content/10.1101/2024.02.12.579963v1) - Recent developments in real-time decision making for cryo-EM workflows - -These references provide scientific context for the automated decision-making algorithms implemented within the SmartEM framework. diff --git a/webui/src/docs/how-to.mdx b/webui/src/docs/how-to.mdx deleted file mode 100644 index 99ec4d4..0000000 --- a/webui/src/docs/how-to.mdx +++ /dev/null @@ -1,36 +0,0 @@ -# How-to Guides - -Practical step-by-step guides for the more experienced user. - -## Getting Started -- [Run Backend](/docs/how-to/run-backend) - Start the SmartEM backend services -- [Run Agent](/docs/how-to/run-agent) - Run the SmartEM agent -- [Run Container](/docs/how-to/run-container) - Run services in containers - -## Development -- [Run E2E Dev Simulation](/docs/how-to/run-e2e-dev-simulation) - End-to-end development testing -- [Development Tools](/docs/how-to/development-tools) - Available development tooling -- [Troubleshoot CLI](/docs/how-to/troubleshoot-cli) - CLI troubleshooting guide -- [Generate Docs](/docs/how-to/generate-docs) - Build documentation locally - -## Configuration -- [Configure Environment Variables](/docs/how-to/configure-environment-variables) - Environment setup -- [Configure Logging](/docs/how-to/configure-logging) - Logging configuration -- [Container User Configuration](/docs/how-to/container-user-configuration) - Container user setup - -## Deployment -- [Containerization](/docs/how-to/containerization) - Build container images -- [Deploy Kubernetes](/docs/how-to/deploy-kubernetes) - Kubernetes deployment guide -- [Manage Kubernetes Secrets](/docs/how-to/manage-kubernetes-secrets) - Secrets management - -## Database -- [Database Migrations](/docs/how-to/database-migrations) - Alembic migration guide - -## APIs & Integration -- [Use API Documentation](/docs/how-to/use-api-documentation) - Interactive API docs -- [Use HTTP API Client](/docs/how-to/use-core-http-api-client) - Core HTTP client usage -- [Use MCP Interface](/docs/how-to/use-mcp-interface) - MCP interface guide - -## Contributing -- [Contribute](/docs/how-to/contribute) - Contribution guidelines -- [Sync GitHub Labels](/docs/how-to/sync-github-labels) - Label synchronization diff --git a/webui/src/docs/how-to/configure-environment-variables.mdx b/webui/src/docs/how-to/configure-environment-variables.mdx deleted file mode 100644 index 5ae3503..0000000 --- a/webui/src/docs/how-to/configure-environment-variables.mdx +++ /dev/null @@ -1,347 +0,0 @@ -# Configure Environment Variables - -This guide explains the different environment configuration files used in the SmartEM Decisions project and their specific purposes. - -## Overview - -The project uses multiple environment configuration patterns for different deployment scenarios: - -| File Pattern | Purpose | Gitignored | Created From | -|--------------|---------|------------|--------------| -| `.env.example` | Template for local development | No (committed) | N/A | -| `.env` | Local development on host OS | Yes | Copy from `.env.example` | -| `.env.example.k8s.development` | Template for K8s development | No (committed) | N/A | -| `.env.k8s.development` | K8s development cluster config | Yes | Copy from `.env.example.k8s.development` | -| `.env.example.k8s.staging` | Template for K8s staging | No (committed) | N/A | -| `.env.k8s.staging` | K8s staging cluster config | Yes | Copy from `.env.example.k8s.staging` | -| `.env.example.k8s.production` | Template for K8s production | No (committed) | N/A | -| `.env.k8s.production` | K8s production cluster config | Yes | Copy from `.env.example.k8s.production` | -| `.env.example.mcp` | Template for MCP configuration | No (committed) | N/A | -| `.env.mcp` | MCP server configuration | Yes | Copy from `.env.example.mcp` | - -## Environment File Types - -### 1. Local Development: `.env` - -**Use Case**: Running backend services directly on your host OS (outside containers) while connecting to K8s infrastructure. - -**When to Use**: -- Day-to-day development with `python -m smartem_backend.consumer` -- Running API server locally with `python -m smartem_backend.api_server` -- E2E testing with `./tools/run-e2e-test.sh` -- Manual debugging and development workflows - -**Setup**: -```bash -cp .env.example .env -# Edit .env with your local credentials -``` - -**Key Configuration**: -```bash -# Connect to K8s infrastructure via NodePorts -POSTGRES_HOST=localhost -POSTGRES_PORT=30432 # K8s NodePort for PostgreSQL -RABBITMQ_HOST=localhost -RABBITMQ_PORT=30672 # K8s NodePort for RabbitMQ - -# Run services on host OS -HTTP_API_PORT=8000 -``` - -**Used By**: -- Python code via `load_dotenv()` in `smartem_backend/consumer.py`, `smartem_backend/utils.py` -- Docker `entrypoint.sh` when running containers outside Kubernetes -- E2E test scripts (`./tools/run-e2e-test.sh`) - -### 2. Kubernetes Development: `.env.k8s.development` - -**Use Case**: Deploying SmartEM to local K8s cluster (k3s) for development. - -**When to Use**: -- Running `./tools/k8s/dev-k8s.sh up` -- Setting up local development cluster with all services - -**Setup**: -```bash -cp .env.example.k8s.development .env.k8s.development -# Add your DOCKER_USERNAME, DOCKER_EMAIL, DOCKER_PASSWORD (GitHub token) -``` - -**Key Configuration**: -```bash -# Docker/GHCR credentials for pulling container images -DOCKER_USERNAME=your-github-username -DOCKER_EMAIL=your-email@example.com -DOCKER_PASSWORD=ghp_your_github_token - -# K8s internal service names (not localhost) -POSTGRES_HOST=postgres-service -POSTGRES_PORT=5432 -RABBITMQ_HOST=rabbitmq-service -RABBITMQ_PORT=5672 - -# Credentials (can be simple for local dev) -POSTGRES_USER=username -POSTGRES_PASSWORD=password -RABBITMQ_USER=username -RABBITMQ_PASSWORD=password -``` - -**Used By**: -- `./tools/k8s/dev-k8s.sh` script to create K8s Secrets and ConfigMaps - -### 3. Kubernetes Staging: `.env.k8s.staging` - -**Use Case**: Deploying to staging Kubernetes cluster. - -**Setup**: -```bash -cp .env.example.k8s.staging .env.k8s.staging -# Add production-like credentials and configuration -``` - -**Key Differences from Development**: -- Real credentials (not test values) -- Production-like service names and ports -- Staging-specific CORS origins -- May use sealed secrets instead of plain credentials - -**Used By**: -- CI/CD pipelines for staging deployments -- `DEPLOY_ENV=staging ./tools/k8s/dev-k8s.sh up` (if adapted for remote clusters) - -### 4. Kubernetes Production: `.env.k8s.production` - -**Use Case**: Deploying to production Kubernetes cluster. - -**Setup**: -```bash -cp .env.example.k8s.production .env.k8s.production -# Add production credentials (should use Sealed Secrets in practice) -``` - -**Security Considerations**: -- Should use Sealed Secrets (encrypted) instead of plain `.env` files -- Credentials should be rotated regularly -- Access to this file should be strictly controlled -- See [Manage Kubernetes Secrets](manage-kubernetes-secrets.md) for best practices - -**Used By**: -- Production CI/CD pipelines -- Production deployment automation - -### 5. MCP Configuration: `.env.mcp` - -**Use Case**: Configuring Model Context Protocol (MCP) server for Claude Code integration. - -**Setup**: -```bash -cp .env.example.mcp .env.mcp -# Configure paths and adapter settings -``` - -**Key Configuration**: -```bash -SMARTEM_MCP_DATA_PATH=/path/to/epu/sessions # Filesystem adapter -SMARTEM_MCP_API_URL=http://backend:8000 # API adapter -SMARTEM_MCP_ADAPTER=filesystem # Adapter selection -SMARTEM_MCP_LOG_LEVEL=INFO -``` - -**Used By**: -- MCP server when run via Claude Code -- See [Use MCP Interface](use-mcp-interface.md) for details - -## Quick Start: First-Time Setup - -For local development on a new machine: - -```bash -# 1. Copy environment templates -cp .env.example .env -cp .env.example.k8s.development .env.k8s.development - -# 2. Edit .env.k8s.development with your GitHub credentials -# Required: DOCKER_USERNAME, DOCKER_EMAIL, DOCKER_PASSWORD -nano .env.k8s.development - -# 3. Start K8s cluster (creates infrastructure) -./tools/k8s/dev-k8s.sh up - -# 4. .env is already configured to connect to K8s NodePorts -# No changes needed unless you want custom ports - -# 5. Test local development -source .venv/bin/activate -python -m smartem_backend.api_server -``` - -## How Environment Variables Are Loaded - -### Python Services - -All Python services use `load_dotenv(override=False)`: - -```python -from dotenv import load_dotenv -load_dotenv(override=False) # Loads .env but doesn't override existing env vars -``` - -**Loading Priority** (first wins): -1. Already exported environment variables (from `export` or `source .env`) -2. Variables in `.env` file (loaded by `load_dotenv()`) -3. Hardcoded defaults in code (if any) - -**Important for Testing**: -When running E2E tests or manual services, you must export variables BEFORE starting services: - -```bash -# Export all variables from .env -set -a && source .env && set +a - -# Now start services -python -m smartem_backend.api_server -python -m smartem_backend.consumer -``` - -### Docker Containers - -Container behaviour depends on runtime environment: - -```bash -# entrypoint.sh checks for Kubernetes -if [ -z "$KUBERNETES_SERVICE_HOST" ]; then - # Running locally - source .env file - source .env -else - # Running in K8s - use ConfigMap/Secrets - # Variables already injected by K8s -fi -``` - -### Kubernetes Deployments - -When running in Kubernetes: -- ConfigMaps provide non-sensitive configuration (hosts, ports, URLs) -- Secrets provide sensitive data (passwords, tokens) -- `.env` files are NOT used inside containers -- All configuration comes from K8s resources created by `dev-k8s.sh` - -## Common Scenarios - -### Scenario 1: Local Development with K8s Infrastructure - -**Goal**: Run backend/agent on host OS, connect to K8s database/RabbitMQ. - -```bash -# 1. Start K8s cluster -./tools/k8s/dev-k8s.sh up - -# 2. Use .env (already configured for NodePorts) -cat .env -# POSTGRES_HOST=localhost -# POSTGRES_PORT=30432 -# RABBITMQ_HOST=localhost -# RABBITMQ_PORT=30672 - -# 3. Run services on host -source .venv/bin/activate -python -m smartem_backend.api_server # Uses .env via load_dotenv() -python -m smartem_backend.consumer -vv # Uses .env via load_dotenv() -``` - -### Scenario 2: Full K8s Deployment - -**Goal**: Everything runs in K8s (no host OS services). - -```bash -# 1. Configure K8s environment -cp .env.example.k8s.development .env.k8s.development -# Edit with your credentials - -# 2. Deploy -./tools/k8s/dev-k8s.sh up - -# 3. All services run in pods -kubectl get pods -n smartem-decisions -# Access via NodePorts: -# - API: http://localhost:30080 -# - Adminer: http://localhost:30808 -# - RabbitMQ UI: http://localhost:30673 -``` - -### Scenario 3: E2E Testing - -**Goal**: Run automated E2E tests with recorded microscope data. - -```bash -# 1. Ensure .env exists and is configured -test -f .env || cp .env.example .env - -# 2. Start K8s cluster -./tools/k8s/dev-k8s.sh up - -# 3. Run E2E test (script loads .env automatically) -./tools/run-e2e-test.sh - -# The script does: -# - source .env (loads variables) -# - Starts services on host OS -# - Runs recorded data playback -# - Collects results -``` - -## Troubleshooting - -### Issue: Services can't connect to database - -**Symptom**: `psycopg2.OperationalError: could not connect to server` - -**Check**: -1. Is K8s cluster running? `kubectl get pods -n smartem-decisions` -2. Are NodePorts accessible? `curl http://localhost:30432` (should refuse connection but not timeout) -3. Is `.env` configured correctly? - ```bash - grep POSTGRES .env - # Should show localhost:30432 for K8s NodePort - ``` - -### Issue: Docker password invalid in dev-k8s.sh - -**Symptom**: `DOCKER_PASSWORD does not appear to be a valid GitHub token` - -**Solution**: -```bash -# Generate token: https://github.com/settings/tokens -# Or use GitHub CLI -gh auth token - -# Add to .env.k8s.development -DOCKER_PASSWORD=ghp_your_token_here -``` - -### Issue: Services using wrong database - -**Symptom**: Services connect to different database than expected. - -**Check loading priority**: -```bash -# Unset ALL environment variables first -unset POSTGRES_HOST POSTGRES_PORT POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD - -# Then load .env -set -a && source .env && set +a - -# Verify variables -env | grep POSTGRES -``` - -## Related Documentation - -- [Run Backend Services](run-backend.md) - Starting backend API and consumer -- [Run E2E Development Simulation](run-e2e-dev-simulation.md) - E2E testing workflows -- [Deploy to Kubernetes](deploy-kubernetes.md) - K8s deployment guide -- [Manage Kubernetes Secrets](manage-kubernetes-secrets.md) - Sealed Secrets and security -- [Database Migrations](database-migrations.md) - Alembic migration workflow -- [Use MCP Interface](use-mcp-interface.md) - Claude Code MCP integration diff --git a/webui/src/docs/how-to/configure-logging.mdx b/webui/src/docs/how-to/configure-logging.mdx deleted file mode 100644 index 4f47016..0000000 --- a/webui/src/docs/how-to/configure-logging.mdx +++ /dev/null @@ -1,183 +0,0 @@ -# Configure Logging and Verbosity - -All SmartEM services support configurable logging levels to help with debugging and reduce noise in production environments. - -## Command Line Verbosity - -Use the `-v` and `-vv` flags to control verbosity across all SmartEM components: - -### Backend Services -```bash -# ERROR level only (default - minimal output) -python -m smartem_backend.consumer -python -m smartem_backend.api_server - -# INFO level and above (-v flag) -python -m smartem_backend.consumer -v -python -m smartem_backend.api_server -v - -# DEBUG level and above (-vv flag - most verbose) -python -m smartem_backend.consumer -vv -python -m smartem_backend.api_server -vv -``` - -### SmartEM Agent CLI -All agent commands support consistent verbosity flags: - -```bash -# ERROR level only (default - minimal output) -python -m smartem_agent watch /path/to/data -python -m smartem_agent validate /path/to/data -python -m smartem_agent parse dir /path/to/data - -# INFO level and above (-v flag) -python -m smartem_agent watch /path/to/data -v -python -m smartem_agent validate /path/to/data -v -python -m smartem_agent parse session /path/to/EpuSession.dm -v - -# DEBUG level and above (-vv flag - most verbose) -python -m smartem_agent watch /path/to/data -vv -python -m smartem_agent parse dir /path/to/data -vv -``` - -### Agent-Specific Logging Features - -The watch command provides additional logging controls: - -```bash -# Custom log file and interval -python -m smartem_agent watch /data \ - --log-file /var/log/smartem/agent.log \ - --log-interval 30.0 \ - -v - -# Real-time communication logging -python -m smartem_agent watch /data \ - --agent-id microscope-01 \ - --session-id session-123 \ - --heartbeat-interval 60 \ - -vv # Shows heartbeat and SSE communication details -``` - -## Environment Variable Control - -For the HTTP API, you can also control logging via environment variables: - -```bash -# Set log level via environment variable (equivalent to -v/-vv flags) -SMARTEM_LOG_LEVEL=ERROR python -m smartem_backend.api_server -SMARTEM_LOG_LEVEL=INFO python -m smartem_backend.api_server -SMARTEM_LOG_LEVEL=DEBUG python -m smartem_backend.api_server -``` - -## Log Levels - -- **ERROR** (default): Only critical errors are shown -- **INFO** (`-v`): Informational messages, warnings, and errors -- **DEBUG** (`-vv`): All messages including detailed debugging information - -## Log Content by Component - -### Backend Services (ERROR level) -- Database connection errors -- API startup failures -- Critical system errors - -### Backend Services (INFO level) -- Service startup and shutdown -- API request summaries -- Database connection status -- Configuration information - -### Backend Services (DEBUG level) -- Individual API request details -- Database query execution -- Message queue operations -- Detailed error stack traces - -### SmartEM Agent (ERROR level) -- File access permission errors -- API connection failures -- Critical parsing errors - -### SmartEM Agent (INFO level) -- File detection and processing -- Grid square and foil hole creation -- API communication status -- SSE connection events -- Heartbeat transmission status - -### SmartEM Agent (DEBUG level) -- Individual file parsing details -- Filesystem event monitoring -- Detailed API request/response data -- SSE message content -- Heartbeat timing information -- Connection retry attempts - -## Production Logging Recommendations - -### Development Environment -```bash -# Use DEBUG level for comprehensive troubleshooting -python -m smartem_agent watch /data -vv -python -m smartem_backend.api_server -vv -``` - -### Testing Environment -```bash -# Use INFO level for verification without noise -python -m smartem_agent watch /data -v -python -m smartem_backend.api_server -v -``` - -### Production Environment -```bash -# Use ERROR level (default) for minimal logging overhead -python -m smartem_agent watch /data -python -m smartem_backend.api_server - -# Or INFO level with log rotation for operational monitoring -python -m smartem_agent watch /data \ - --log-file /var/log/smartem/agent.log \ - --log-interval 60.0 \ - -v -``` - -## Log File Management - -### Structured Logging -The agent's `--log-file` parameter creates structured JSON logs suitable for analysis: - -```json -{ - "timestamp": "2024-01-15T14:30:22.123456", - "event_count": 5, - "events": [ - { - "timestamp": "2024-01-15T14:30:22.120000", - "event_type": "created", - "source_path": "/data/Grid_001/EpuSession.dm", - "relative_path": "Grid_001/EpuSession.dm", - "size": 2048, - "modified": "2024-01-15T14:30:22.119000" - } - ] -} -``` - -### Log Rotation and Management -```bash -# Use logrotate or similar tools for production log management -# Example logrotate configuration: -/var/log/smartem/agent.log { - daily - rotate 30 - compress - delaycompress - notifempty - missingok -} -``` - -This verbosity control system helps reduce log noise during normal operation while providing detailed output when troubleshooting issues. The structured logging format facilitates automated monitoring and analysis of system behaviour. diff --git a/webui/src/docs/how-to/container-user-configuration.mdx b/webui/src/docs/how-to/container-user-configuration.mdx deleted file mode 100644 index 374d8b1..0000000 --- a/webui/src/docs/how-to/container-user-configuration.mdx +++ /dev/null @@ -1,425 +0,0 @@ -# Container User Configuration - -This guide explains how to configure the SmartEM Decisions container to run as a non-root user with specific UID/GID, which is essential for certain deployment scenarios at Diamond Light Source (DLS). - -## Overview - -The SmartEM Decisions Dockerfile supports two distinct operational modes: - -1. **Default Mode (Root User)**: For CI/CD pipelines and local development -2. **Custom User Mode**: For production deployments requiring filesystem access with specific permissions - -## Why Non-Root Users? - -At Diamond Light Source, the HTTP API service needs to access microscopy images stored on the `/dls` filesystem. This filesystem: - -- Contains electron microscopy data from EPU sessions -- Cannot be mounted in containers with root privileges due to security policies -- Requires the container to run with a specific UID/GID that has read permissions -- Is accessed by image serving endpoints in the HTTP API - -Without proper user configuration, the container cannot access `/dls` and image serving endpoints will fail. - -## Build Arguments - -The Dockerfile accepts three build arguments that control user/group creation: - -| Argument | Default | Description | -|----------|---------|-------------| -| `groupid` | `0` | Group ID for the container user. When set to `0`, no custom user is created (runs as root). | -| `userid` | `0` | User ID for the container user. Should match the required UID for filesystem access. | -| `groupname` | `root` | Name for both the group and user. Used for identification in container processes. | - -## Default Behavior: Root User - -When build arguments are not specified (or explicitly set to defaults), the container runs as root: - -```bash -# Build with defaults (runs as root) -docker build -t smartem-decisions . - -# This is equivalent to: -docker build \ - --build-arg groupid=0 \ - --build-arg userid=0 \ - --build-arg groupname=root \ - -t smartem-decisions . -``` - -**Use cases for root mode:** -- CI/CD pipelines (GitHub Actions, GitLab CI) -- Local development environments -- Environments without specific filesystem permission requirements -- Testing and debugging - -**Implications:** -- Container has full privileges -- No user creation step is executed -- All files owned by root (UID 0, GID 0) -- Cannot mount DLS filesystem in production - -## Custom User Mode: DLS Deployment - -For production deployment at DLS, build the container with specific UID/GID: - -```bash -# Build with custom user (example values) -docker build \ - --build-arg groupid=1000 \ - --build-arg userid=1000 \ - --build-arg groupname=smartem \ - -t smartem-decisions:dls . -``` - -**What happens during build:** -1. A group is created with the specified `groupid` and `groupname` -2. A user is created with the specified `userid`, belonging to the group -3. All application files (`/venv`, `/app`, `/entrypoint.sh`) are set to be owned by this user -4. The container will execute processes as this user (not root) - -**Use cases for custom user mode:** -- DLS production/staging deployments -- Any environment requiring specific filesystem permissions -- Security-hardened deployments following least-privilege principles - -**Implications:** -- Container runs with limited privileges -- Can access `/dls` filesystem when mounted with matching permissions -- Image serving endpoints function correctly -- More secure than running as root - -## Volume Mounting Considerations - -### The /dls Directory - -The `/dls` directory is **not** created in the Docker image. It should be mounted at runtime: - -```bash -# Mount /dls directory (example) -docker run -v /path/to/dls:/dls smartem-decisions:dls -``` - -In Kubernetes, this is typically done via a PersistentVolume or hostPath mount: - -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: smartem-http-api -spec: - template: - spec: - containers: - - name: smartem-http-api - image: ghcr.io/diamondlightsource/smartem-decisions:dls - volumeMounts: - - name: dls-data - mountPath: /dls - readOnly: true - volumes: - - name: dls-data - hostPath: - path: /dls - type: Directory -``` - -### When /dls is Not Mounted - -If the `/dls` directory is not mounted or does not exist: - -- The container will start normally -- Most API endpoints will function correctly -- Image serving endpoints will return 404 errors when image paths reference `/dls` -- Error messages will indicate that the file cannot be found - -This is by design - the container is operational without `/dls`, but image serving functionality is unavailable. - -### Optional Mount Strategy - -The `/dls` mount should be considered **optional for development** but **required for production**: - -- **Development/Testing**: Run without `/dls` mount for testing non-image features -- **Staging**: Mount a subset of `/dls` or test data directory -- **Production**: Mount full `/dls` filesystem with proper permissions - -## Image Serving Endpoints - -The HTTP API provides endpoints for serving microscopy images: - -- `GET /grids/{grid_uuid}/atlas_image` - Serve grid atlas images -- `GET /gridsquares/{gridsquare_uuid}/gridsquare_image` - Serve grid square images - -These endpoints: -1. Query the database for image file paths -2. Read image files from the filesystem (typically `/dls`) -3. Process and return images in PNG format - -**Requirements:** -- Container must run as user with read access to image files -- Image paths in database must be accessible by the container user -- Filesystem must be mounted at the correct path - -**Error handling:** -- If image path is not in database: Returns 404 "Grid square image unknown" -- If file doesn't exist: Returns 404 or file system error -- If permissions denied: Returns 500 error with permission details - -## Security Considerations - -### Principle of Least Privilege - -Running containers as non-root is a security best practice: - -- **Risk Reduction**: Limited damage if container is compromised -- **Policy Compliance**: Many organizations require non-root containers -- **Audit Trail**: Clear user identity in logs and process lists - -### Choosing UID/GID - -When selecting UID/GID for custom user mode: - -1. **Match Filesystem Permissions**: Use UID/GID that has read access to required files -2. **Avoid System IDs**: Don't use UIDs below 1000 (reserved for system users) -3. **Document Values**: Keep a record of which UID/GID is used in each environment -4. **Consistency**: Use the same UID/GID across all pods in the same environment - -### File Ownership - -All files in the container are owned by the specified user: - -```bash -# Inside container running as custom user -ls -la /app -# Output: drwxr-xr-x 2 smartem smartem 4096 Oct 7 12:00 . - -ls -la /venv -# Output: drwxr-xr-x 2 smartem smartem 4096 Oct 7 12:00 . -``` - -This ensures the application can read its own files while running as non-root. - -## Building for Different Environments - -### Local Development - -```bash -# Simple build for local development -docker build -t smartem-decisions:dev . - -# Run with local database -docker run -p 8000:8000 \ - -e ROLE=api \ - -e POSTGRES_HOST=host.docker.internal \ - smartem-decisions:dev -``` - -### CI/CD Pipeline - -```bash -# GitHub Actions (runs as root by default) -docker build -t smartem-decisions:$GITHUB_SHA . -docker push ghcr.io/diamondlightsource/smartem-decisions:$GITHUB_SHA -``` - -### Staging Environment - -```bash -# Build with staging UID/GID -docker build \ - --build-arg groupid=1001 \ - --build-arg userid=1001 \ - --build-arg groupname=smartem-staging \ - -t smartem-decisions:staging . - -# Tag and push -docker tag smartem-decisions:staging \ - ghcr.io/diamondlightsource/smartem-decisions:staging -docker push ghcr.io/diamondlightsource/smartem-decisions:staging -``` - -### Production Environment - -```bash -# Build with production UID/GID (example values) -docker build \ - --build-arg groupid=5000 \ - --build-arg userid=5000 \ - --build-arg groupname=smartem \ - -t smartem-decisions:production . - -# Tag and push -docker tag smartem-decisions:production \ - ghcr.io/diamondlightsource/smartem-decisions:production -docker push ghcr.io/diamondlightsource/smartem-decisions:production -``` - -## Kubernetes Deployment Configuration - -### Example: HTTP API with Custom User - -If you've built the image with custom UID/GID, deploy it in Kubernetes: - -```yaml -apiVersion: apps/v1 -kind: Deployment -metadata: - name: smartem-http-api - namespace: smartem-decisions-production -spec: - replicas: 3 - selector: - matchLabels: - app: smartem-http-api - template: - metadata: - labels: - app: smartem-http-api - spec: - # Optional: Explicitly set security context to match build args - securityContext: - runAsUser: 5000 - runAsGroup: 5000 - fsGroup: 5000 - - containers: - - name: smartem-http-api - image: ghcr.io/diamondlightsource/smartem-decisions:production - - # Mount /dls filesystem - volumeMounts: - - name: dls-data - mountPath: /dls - readOnly: true - - env: - - name: ROLE - value: "api" - # ... other environment variables ... - - volumes: - - name: dls-data - hostPath: - path: /dls - type: Directory -``` - -### Security Context in Kubernetes - -While the Dockerfile sets ownership, Kubernetes `securityContext` provides additional enforcement: - -- `runAsUser`: Ensures container runs as specific UID -- `runAsGroup`: Sets primary group ID -- `fsGroup`: Sets group ownership for mounted volumes -- `runAsNonRoot`: Enforces non-root execution (Kubernetes validates) - -Example with security context: - -```yaml -spec: - template: - spec: - securityContext: - runAsNonRoot: true - runAsUser: 5000 - runAsGroup: 5000 - fsGroup: 5000 - containers: - - name: smartem-http-api - image: ghcr.io/diamondlightsource/smartem-decisions:production - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL -``` - -## Troubleshooting - -### Image Serving Endpoints Return 404 - -**Symptoms:** -- API starts successfully -- Most endpoints work -- Image endpoints return 404 - -**Possible causes:** -1. `/dls` not mounted -2. Image paths in database incorrect -3. Permission denied (check with 403/500 errors) - -**Solution:** -```bash -# Check if /dls is mounted -kubectl exec -it -- ls -la /dls - -# Check container user -kubectl exec -it -- id - -# Check file permissions -kubectl exec -it -- ls -la /dls/path/to/images -``` - -### Permission Denied Errors - -**Symptoms:** -- Container starts but cannot read files -- Errors like "Permission denied" in logs - -**Possible causes:** -1. UID/GID mismatch between container and filesystem -2. Files owned by different user - -**Solution:** -```bash -# Verify container UID/GID -docker run --rm smartem-decisions:dls id -# Output: uid=5000(smartem) gid=5000(smartem) groups=5000(smartem) - -# Verify filesystem permissions -ls -lan /path/to/dls/data -# Ensure UID 5000 has read access -``` - -### Container Fails to Start - -**Symptoms:** -- Container exits immediately -- Error about user creation - -**Possible causes:** -1. Invalid UID/GID values -2. Conflicting user/group names - -**Solution:** -```bash -# Check build arguments -docker inspect smartem-decisions:dls | grep -A 5 "Args" - -# Rebuild with correct arguments -docker build --build-arg groupid=5000 --build-arg userid=5000 \ - --build-arg groupname=smartem -t smartem-decisions:dls . -``` - -## Best Practices - -1. **Document UID/GID Choices**: Keep a record of which UID/GID is used in each environment -2. **Use Consistent Values**: Don't change UID/GID between builds for the same environment -3. **Test Both Modes**: Ensure the application works in both root and non-root modes -4. **Validate Permissions**: Before deploying, verify the container user can access required files -5. **Plan Volume Mounts**: Design mount strategy before building custom images -6. **Monitor Logs**: Watch for permission-related errors during deployment -7. **Use Security Context**: Combine Dockerfile user config with Kubernetes securityContext -8. **Graceful Degradation**: Ensure the application handles missing `/dls` gracefully - -## Related Documentation - -- [Run in a Container](run-container.md) - Basic container usage -- [Kubernetes Deployment](deploy-kubernetes.md) - Full deployment guide -- [Containerization](containerization.md) - Building and pushing images -- [Managing Kubernetes Secrets](manage-kubernetes-secrets.md) - Secure credential management - -## References - -- [Docker USER instruction](https://docs.docker.com/engine/reference/builder/#user) -- [Kubernetes Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) -- [Best practices for running containers](https://docs.docker.com/develop/dev-best-practices/) diff --git a/webui/src/docs/how-to/containerization.mdx b/webui/src/docs/how-to/containerization.mdx deleted file mode 100644 index c7bb3da..0000000 --- a/webui/src/docs/how-to/containerization.mdx +++ /dev/null @@ -1,123 +0,0 @@ -# Containerization - -This guide covers building and managing SmartEM Decisions container images. - -## Docker/Podman Operations - -### Basic Build (Default Configuration) - -Build the container with default settings (runs as root): - -```bash -# Docker -docker build -t smartem-decisions . - -# Podman -podman build --format docker -t smartem-decisions . -``` - -### Build with Custom User (DLS Deployment) - -For production deployment at Diamond Light Source, build with custom UID/GID to enable `/dls` filesystem access: - -```bash -# Docker -docker build \ - --build-arg groupid=1000 \ - --build-arg userid=1000 \ - --build-arg groupname=smartem \ - -t smartem-decisions:dls . - -# Podman -podman build --format docker \ - --build-arg groupid=1000 \ - --build-arg userid=1000 \ - --build-arg groupname=smartem \ - -t smartem-decisions:dls . -``` - -See [Container User Configuration](container-user-configuration.md) for detailed information about build arguments and deployment scenarios. - -### Running Containers - -```bash -# Docker - run API service -docker run -p 8000:8000 -e ROLE=api smartem-decisions - -# Podman - run API service -podman run -p 8000:8000 -e ROLE=api localhost/smartem-decisions -``` - -### Cleanup - -```bash -# Docker -docker image rm smartem-decisions -f - -# Podman -podman image rm localhost/smartem-decisions -f -``` - -## Tagging and Pushing Images - -### GitHub Container Registry (Current) - -```bash -# Tag the image -docker tag smartem-decisions ghcr.io/diamondlightsource/smartem-decisions:latest - -# Login to GitHub Container Registry -echo $GITHUB_TOKEN | docker login ghcr.io -u USERNAME --password-stdin - -# Push the image -docker push ghcr.io/diamondlightsource/smartem-decisions:latest -``` - -### Tagging Strategies - -```bash -# Development/staging build -docker tag smartem-decisions ghcr.io/diamondlightsource/smartem-decisions:dev -docker tag smartem-decisions ghcr.io/diamondlightsource/smartem-decisions:staging - -# Production build with version -docker tag smartem-decisions ghcr.io/diamondlightsource/smartem-decisions:1.0.0 -docker tag smartem-decisions ghcr.io/diamondlightsource/smartem-decisions:latest - -# Production build with custom user -docker tag smartem-decisions:dls ghcr.io/diamondlightsource/smartem-decisions:production -``` - -## Multi-Stage Build Architecture - -The Dockerfile uses a multi-stage build process: - -1. **developer** stage: Base Python image with system dependencies -2. **build** stage: Installs Python packages and application code -3. **runtime** stage: Slim image with only runtime dependencies and built application - -This approach: -- Minimizes final image size -- Separates build tools from runtime environment -- Enables efficient layer caching - -## Build Arguments Reference - -| Argument | Default | Purpose | -|----------|---------|---------| -| `PYTHON_VERSION` | `3.12` | Python version for base images | -| `groupid` | `0` | Group ID for container user (0 = root) | -| `userid` | `0` | User ID for container user (0 = root) | -| `groupname` | `root` | Name for group and user | - -## Related Documentation - -- [Container User Configuration](container-user-configuration.md) - Detailed guide on UID/GID configuration -- [Run in a Container](run-container.md) - Using pre-built containers -- [Kubernetes Deployment](deploy-kubernetes.md) - Deploying to Kubernetes - -## References - -- [GitHub Container Registry](https://ghcr.io/diamondlightsource/smartem-decisions) -- [Docker Multi-stage Builds](https://docs.docker.com/build/building/multi-stage/) -- [Podman Documentation](https://docs.podman.io/) diff --git a/webui/src/docs/how-to/contribute.mdx b/webui/src/docs/how-to/contribute.mdx deleted file mode 100644 index 22749c3..0000000 --- a/webui/src/docs/how-to/contribute.mdx +++ /dev/null @@ -1,27 +0,0 @@ -# Contributing - -Thank you for your interest in contributing to SmartEM! - -## Getting Started - -1. Fork the repository -2. Clone your fork locally -3. Create a feature branch -4. Make your changes -5. Submit a pull request - -## Code Standards - -- Use British English in documentation -- No emojis in code or docs (Windows build compatibility) -- Follow existing patterns and conventions -- Add tests for new functionality - -## Documentation - -- Follow the [Diataxis](https://diataxis.fr/) framework -- Use ADRs for significant architectural decisions - -## Questions? - -Open an issue on GitHub for questions or suggestions. diff --git a/webui/src/docs/how-to/database-migrations.mdx b/webui/src/docs/how-to/database-migrations.mdx deleted file mode 100644 index 33dd5ee..0000000 --- a/webui/src/docs/how-to/database-migrations.mdx +++ /dev/null @@ -1,280 +0,0 @@ -# Database Migrations - -This guide explains how to use database migrations in the smartem-decisions project. - -## Overview - -We use [Alembic](https://alembic.sqlalchemy.org/) for database schema versioning and migrations. Alembic integrates with our SQLModel-based database models to provide: - -- **Schema versioning**: Track database changes over time -- **Automated migration generation**: Generate migrations from model changes -- **Rollback support**: Safely revert database changes -- **Environment-specific deployments**: Apply migrations across development, staging, and production - -## Prerequisites - -Ensure you have the backend dependencies installed: - -```bash -pip install -e .[backend] -``` - -This includes Alembic as a dependency. - -## Basic Usage - -### Initial Database Setup - -For a completely new database, run the following command to apply all migrations from scratch: - -```bash -python -m alembic upgrade head -``` - -This will: -1. Create the Alembic version tracking table -2. Apply migration 001: Create user preferences table -3. Apply baseline migration (6e6302f1ccb6): Create all core SmartEM schema tables (acquisition, grid, gridsquare, micrograph, foilhole, atlas, quality prediction models, etc.) -4. Apply migration 002: Add performance indexes for acquisition datetime queries -5. Apply remaining migrations including agent communication tables - -### Running Migrations - -Apply all pending migrations to bring your database up to the latest schema: - -```bash -python -m alembic upgrade head -``` - -### Checking Migration Status - -See the current database version: - -```bash -python -m alembic current -``` - -View migration history: - -```bash -python -m alembic history --verbose -``` - -### Rolling Back Migrations - -Downgrade to a specific revision: - -```bash -python -m alembic downgrade -``` - -Rollback one migration: - -```bash -python -m alembic downgrade -1 -``` - -## Creating New Migrations - -### Auto-generating Migrations - -When you modify SQLModel classes in `src/smartem_backend/model/database.py`, generate a migration automatically: - -```bash -python -m alembic revision --autogenerate -m "Add new field to Grid model" -``` - -**Important**: Always review auto-generated migrations before applying them. Alembic may not detect all changes (like column renames or complex constraints). - -### Manual Migrations - -For data migrations or complex schema changes, create an empty migration: - -```bash -python -m alembic revision -m "Seed initial user data" -``` - -Edit the generated file in `src/smartem_backend/migrations/versions/` to add your custom logic. - -## Migration Examples - -### Structure Changes - -```python -def upgrade() -> None: - op.create_table('user_preferences', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('user_id', sa.String(), nullable=False), - sa.Column('preference_key', sa.String(255), nullable=False), - sa.Column('preference_value', postgresql.JSONB(), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - -def downgrade() -> None: - op.drop_table('user_preferences') -``` - -### Index Changes - -```python -def upgrade() -> None: - op.create_index( - 'idx_gridsquare_acquisition_datetime_status', - 'gridsquare', - ['acquisition_datetime', 'status'], - postgresql_using='btree' - ) - -def downgrade() -> None: - op.drop_index('idx_gridsquare_acquisition_datetime_status') -``` - -### Data Changes - -```python -from sqlalchemy.sql import table, column - -def upgrade() -> None: - user_preferences = table('user_preferences', - column('user_id', sa.String), - column('preference_key', sa.String), - column('preference_value', sa.JSON) - ) - - op.bulk_insert(user_preferences, [ - { - 'user_id': 'system', - 'preference_key': 'default_settings', - 'preference_value': {'theme': 'light', 'page_size': 50} - } - ]) - -def downgrade() -> None: - op.execute("DELETE FROM user_preferences WHERE user_id = 'system'") -``` - -## Best Practices - -### Migration Safety - -1. **Always backup production data** before running migrations -2. **Test migrations** on a copy of production data first -3. **Review auto-generated migrations** carefully before applying -4. **Write reversible migrations** with proper downgrade logic - -### Schema Changes - -1. **Add columns as nullable first**, then make them non-nullable in a separate migration if needed -2. **Use separate migrations** for structure and data changes -3. **Include appropriate indexes** for performance-critical fields -4. **Document breaking changes** in migration comments - -### Development Workflow - -1. **Create feature branch** for database changes -2. **Generate migration** after modifying models -3. **Test migration** locally (up and down) -4. **Review migration code** before committing -5. **Coordinate with team** for schema changes affecting multiple developers - -## Deployment - -### Production Deployment - -```bash -# 1. Backup database -pg_dump smartem_production > backup_$(date +%Y%m%d_%H%M%S).sql - -# 2. Apply migrations -python -m alembic upgrade head - -# 3. Verify application works -# 4. Clean up old backups (optional) -``` - -### Staging Environment - -```bash -# Apply migrations to staging first -python -m alembic upgrade head - -# Test application functionality -# Run integration tests -``` - -## Troubleshooting - -### Common Issues - -**Migration conflicts**: When multiple developers create migrations simultaneously: -```bash -# Resolve by creating a merge migration -python -m alembic merge -m "Merge migrations" -``` - -**Failed migration**: If a migration fails partway through: -```bash -# Check current state -python -m alembic current - -# Fix data/schema issues manually -# Mark migration as complete (if safe) -python -m alembic stamp -``` - -**Model out of sync**: When models don't match database: -```bash -# Generate migration to sync -python -m alembic revision --autogenerate -m "Sync models with database" -``` - -### Migration-Specific Issues - -**Missing baseline schema**: If you encounter errors like "relation 'gridsquare' does not exist" when running migrations: - -This indicates that migration 002 is trying to create indexes on tables that don't exist yet. This happens when the baseline schema migration is missing or not properly applied. The fix is to ensure the baseline migration (6e6302f1ccb6) is applied before migration 002. - -```bash -# Solution: Run the complete migration sequence from scratch -python -m alembic downgrade base -python -m alembic upgrade head -``` - -**Enum type conflicts**: If you encounter "type already exists" errors: - -This can happen when enum types exist from previous schema creation attempts but don't match the migration expectations. - -```bash -# Clean up existing enum types and restart migrations -python -m alembic downgrade base -# Then apply migrations fresh -python -m alembic upgrade head -``` - -### Database Connection Issues - -The migration system uses your existing database connection configuration from `smartem_backend.utils.setup_postgres_connection()`. - -Ensure your environment variables are set: -- Database connection parameters -- PostgreSQL credentials -- Network access to database server - -## Files and Structure - -``` -├── alembic.ini # Alembic configuration -├── src/smartem_backend/migrations/ -│ ├── env.py # Migration environment -│ ├── script.py.mako # Migration template -│ └── versions/ # Individual migration files -│ ├── 001_add_user_preferences.py -│ ├── 002_add_acquisition_time_index.py -│ └── 003_seed_system_config.py -``` - -## Further Reading - -- [Alembic Documentation](https://alembic.sqlalchemy.org/) -- [SQLModel Documentation](https://sqlmodel.tiangolo.com/) -- [PostgreSQL Documentation](https://www.postgresql.org/docs/) diff --git a/webui/src/docs/how-to/deploy-kubernetes.mdx b/webui/src/docs/how-to/deploy-kubernetes.mdx deleted file mode 100644 index 80accdb..0000000 --- a/webui/src/docs/how-to/deploy-kubernetes.mdx +++ /dev/null @@ -1,77 +0,0 @@ -# Kubernetes Deployment - -This directory contains Kubernetes deployment configurations for SmartEM Backend across different environments. - -## Quick Start (Development) - -For local development, use the convenient script that provides a docker-compose-like experience: - -```bash -# Start the development environment (equivalent to docker-compose up -d) -./tools/k8s/dev-k8s.sh - -# Check status (equivalent to docker ps) -./tools/k8s/dev-k8s.sh status - -# View logs for a service -./tools/k8s/dev-k8s.sh logs smartem-http-api - -# Stop the environment (equivalent to docker-compose down) -./tools/k8s/dev-k8s.sh down - -# Restart everything -./tools/k8s/dev-k8s.sh restart - -# Get help -./tools/k8s/dev-k8s.sh --help -``` - -### Access URLs -Once the environment is running, you can access: -- **Adminer (Database UI)**: http://localhost:30808 -- **RabbitMQ Management**: http://localhost:30673 -- **SmartEM Backend HTTP API**: http://localhost:30080/health -- **API Documentation**: http://localhost:30080/docs - -> **Note**: The script automatically handles GitHub Container Registry authentication and waits for all pods to be ready. - -## Kubernetes Structure - -``` -k8s/ -├── environments/ -│ ├── development/ # Local development (k3s) -│ ├── staging/ # Staging environment (pollux) -│ └── production/ # Production environment (argos?) -└── README.md -``` - -## Security: Sealed Secrets - -The project uses [Bitnami Sealed Secrets](https://sealed-secrets.netlify.app/) for secure credential management. Before -deploying to any environment, you must generate the appropriate sealed secrets: - -### Generate Secrets for Development - -```bash -# Auto-generate secure credentials for development -./tools/k8s/generate-sealed-secrets.sh development -``` - -### Generate Secrets for Production - -```bash -# Interactive credential input for production security -./tools/k8s/generate-sealed-secrets.sh production -``` - -Sealed secrets are encrypted with the cluster's public key and safe to commit to version control. The sealed-secrets -controller automatically decrypts them into regular Kubernetes secrets that applications can use. - -> **Security Note**: Never commit plain-text secrets to version control. Always use sealed secrets for credential management. - -For comprehensive secret management documentation, see [Managing Kubernetes Secrets](manage-kubernetes-secrets.md). - -## Detailed Documentation - -For detailed Kubernetes deployment instructions, environment configurations, and troubleshooting, see the [k8s directory documentation](k8s/). diff --git a/webui/src/docs/how-to/development-tools.mdx b/webui/src/docs/how-to/development-tools.mdx deleted file mode 100644 index bfce1cb..0000000 --- a/webui/src/docs/how-to/development-tools.mdx +++ /dev/null @@ -1,172 +0,0 @@ -# Development Tools - -Collection of utility tools for development, testing, and maintenance of the SmartEM Decisions project. - -## XML Formatting Tools - -### Format XML Files for Human Readability - -Transform single-line XML and .dm files into human-readable format with proper indentation: - -```bash -# Reformat all .xml and .dm files in a directory recursively -python tools/format_xml.py -r - -# Process multiple directories -python tools/format_xml.py -r \ - ../smartem-decisions-test-datasets/metadata_Supervisor_20250114_220855_23_epuBSAd20_GrOxDDM \ - ../smartem-decisions-test-datasets/metadata_Supervisor_20241220_140307_72_et2_gangshun \ - ../smartem-decisions-test-datasets/metadata_Supervisor_20250108_101446_62_cm40593-1_EPU - -# Display all available options -python tools/format_xml.py --help -``` - -## Data Analysis and Debugging Tools - -### Find Foil Hole Manifest Duplicates - -Identify duplicate foil hole manifests within directory structures to detect data inconsistencies: - -```bash -# Display help and usage information -tools/find_foilhole_duplicates.py --help - -# Example: Search for duplicates in test data -tools/find_foilhole_duplicates.py ./tests/testdata/bi37708-28 -``` - -### File Size Analysis - -List files matching specific patterns, sorted by size for storage analysis: - -```bash -# Find GridSquare files sorted by size (largest first) -rg --files -g 'GridSquare_*.dm' ./tests/testdata/bi37708-28 \ - | xargs -d '\n' ls -lh | sort -k5 -rn | awk '{print $9, $5}' -``` - -## Test Dataset Management - -### File Extension Analysis - -Analyse the composition of test datasets by file type: - -```bash -# Recursively find all distinct file extensions with counts -find . -type f | - sed -E 's/.*\.([^.]+)$/\1/' | - grep -v "/" | - sort | - uniq -c | - sort -nr -``` - -### Dataset Size Reduction - -Reduce test dataset storage requirements whilst maintaining directory structure: - -```bash -# Empty image and data files whilst preserving metadata structure -find . -type f \( -name "*.jpg" -o -name "*.png" -o -name "*.mrc" \) -exec truncate -s 0 {} \; -``` - -**Warning:** This command permanently removes file contents. Use only on test datasets, not production data. - -## Development Monitoring - -### Directory Growth Monitoring - -Monitor directory metrics during data acquisition or processing: - -```bash -# Watch directory size and file count with 1-second updates -watch -n 1 'echo "Size: $(du -sh .)"; echo "Files: $(find . -type f | wc -l)"' -``` - -This tool is particularly useful for monitoring EPU data acquisition progress or debugging processing pipeline performance. - -## Message Testing and Communication Tools - -### External Message Simulator - -Comprehensive CLI tool for simulating external data processing messages that would normally come from ML pipelines and image processing systems: - -```bash -# List all available message types -python tools/external_message_simulator.py list-messages - -# Individual message simulation examples -python tools/external_message_simulator.py motion-correction --foilhole-id "FH_001_001_001" --quality-score 0.85 -python tools/external_message_simulator.py ctf-complete --foilhole-id "FH_001_001_001" --resolution 3.2 -python tools/external_message_simulator.py gridsquare-prediction --gridsquare-id "GS_001_001" --prediction-score 0.85 -python tools/external_message_simulator.py model-update --model-name "ResNet-50" - -# Complete workflow simulation for a single grid square -python tools/external_message_simulator.py workflow-simulation --gridsquare-id "DEV_001" - -# Batch simulation with different quality scenarios -python tools/external_message_simulator.py batch-simulation --gridsquare-count 5 --scenario mixed -python tools/external_message_simulator.py batch-simulation --gridsquare-count 3 --scenario good -python tools/external_message_simulator.py batch-simulation --gridsquare-count 2 --scenario poor -``` - -**Available Message Types:** -- `MOTION_CORRECTION_COMPLETE` - Motion correction processing finished -- `CTF_COMPLETE` - CTF estimation completed -- `PARTICLE_PICKING_COMPLETE` - Particle identification finished -- `PARTICLE_SELECTION_COMPLETE` - Particle quality assessment done -- `GRIDSQUARE_MODEL_PREDICTION` - ML prediction for grid square quality -- `FOILHOLE_MODEL_PREDICTION` - ML prediction for foilhole targeting -- `MODEL_PARAMETER_UPDATE` - ML model parameter updates - -### SSE Client Testing - -Example client for testing agent-backend communication via Server-Sent Events: - -```bash -# Start the SSE client to receive instructions from backend -python tools/sse_client_example.py - -# The client will: -# 1. Auto-create a new session with the backend -# 2. Connect to the SSE stream for real-time instructions -# 3. Acknowledge received instructions -# 4. Measure processing time for performance testing -``` - -This tool is particularly useful for: -- Testing the complete agent-backend communication pipeline -- Validating instruction delivery and acknowledgement mechanisms -- Performance testing of SSE communication -- Development of new agent integrations - -Both tools work together to simulate the complete external data flow into the SmartEM system, enabling comprehensive testing without requiring actual microscopy equipment or external processing systems. - -## Additional Development Commands - -### Pre-commit Workflow - -Maintain code quality during development: - -```bash -# Run pre-commit checks on specific files -pre-commit run --files - -# Run all pre-commit checks -pre-commit run --all-files -``` - -### Testing and Quality Assurance - -```bash -# Run comprehensive test suite -pytest - -# Type checking with pyright -pyright src tests - -# Code formatting and linting -ruff check -ruff format -``` diff --git a/webui/src/docs/how-to/generate-docs.mdx b/webui/src/docs/how-to/generate-docs.mdx deleted file mode 100644 index 7bfe97b..0000000 --- a/webui/src/docs/how-to/generate-docs.mdx +++ /dev/null @@ -1,8 +0,0 @@ -# Generate Documentation - -## Documentation Generation - -```sh -tox -e docs -python -m http.server -d build/html -``` diff --git a/webui/src/docs/how-to/manage-kubernetes-secrets.mdx b/webui/src/docs/how-to/manage-kubernetes-secrets.mdx deleted file mode 100644 index b7e36fb..0000000 --- a/webui/src/docs/how-to/manage-kubernetes-secrets.mdx +++ /dev/null @@ -1,377 +0,0 @@ -# Managing Kubernetes Secrets - -This guide explains how to securely manage secrets in the SmartEM Decisions project using Bitnami Sealed Secrets. - -## Overview - -The SmartEM Decisions project uses [Bitnami Sealed Secrets](https://sealed-secrets.netlify.app/) to securely manage sensitive -configuration data such as database credentials and message queue passwords. Sealed Secrets provide a secure alternative to -storing plain-text secrets in version control. - -### Why Sealed Secrets? - -Sealed Secrets offer several security advantages over traditional Kubernetes secrets: - -- **Version Control Safe**: Sealed secrets are encrypted and safe to commit to Git repositories -- **Asymmetric Encryption**: Uses public/private key cryptography for maximum security -- **Cluster-Specific**: Secrets are encrypted for a specific cluster and cannot be used elsewhere -- **Automatic Decryption**: The sealed-secrets controller automatically decrypts secrets in the cluster -- **Audit Trail**: All secret changes are tracked in version control with proper attribution - -### How It Works - -1. **Public Key Encryption**: Secrets are encrypted using the cluster's public key -2. **Safe Storage**: Encrypted sealed secrets are committed to version control -3. **Automatic Decryption**: The sealed-secrets controller watches for sealed secrets and creates regular Kubernetes secrets -4. **Application Access**: Applications access secrets normally via environment variables or mounted volumes - -## Prerequisites - -Before managing sealed secrets, ensure you have the required tools installed: - -### Required Tools - -- **kubectl**: Kubernetes command-line tool with cluster access -- **kubeseal**: Bitnami Sealed Secrets CLI tool -- **openssl**: For secure password generation (development environments) - -### Installing kubeseal - -```bash -# Download latest release (Linux) -wget https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.18.0/kubeseal-0.18.0-linux-amd64.tar.gz -tar -xvzf kubeseal-0.18.0-linux-amd64.tar.gz -sudo install -m 755 kubeseal /usr/local/bin/kubeseal - -# Verify installation -kubeseal --version -``` - -### Cluster Requirements - -The sealed-secrets controller must be installed in your Kubernetes cluster. For Diamond Light Source clusters, this is -typically pre-installed. Verify the controller is running: - -```bash -kubectl get pods -n kube-system -l name=sealed-secrets-controller -``` - -## Quick Start - -The project includes a convenient script that handles the entire sealed secret generation process: - -```bash -# Generate secrets for development (auto-generated passwords) -./tools/k8s/generate-sealed-secrets.sh development - -# Generate secrets for production (interactive prompts) -./tools/k8s/generate-sealed-secrets.sh production - -# Generate secrets for all environments -./tools/k8s/generate-sealed-secrets.sh all -``` - -## Environment-Specific Workflows - -### Development Environment - -Development environments use automatically generated secure passwords for convenience: - -```bash -./tools/k8s/generate-sealed-secrets.sh development -``` - -This will: -- Generate cryptographically secure random passwords -- Create sealed secrets for the `smartem-decisions` namespace -- Display a summary of generated usernames (passwords remain sealed) -- Update `k8s/environments/development/secrets.yaml` - -### Staging and Production Environments - -Production and staging environments require interactive credential input for security: - -```bash -./tools/k8s/generate-sealed-secrets.sh production -``` - -You'll be prompted to provide: -- PostgreSQL username and password -- RabbitMQ username and password - -**Security Note**: Passwords are entered without echo (not displayed on screen) and never appear in shell history. - -## Manual Secret Management - -For advanced use cases, you can manually create sealed secrets: - -### 1. Create Temporary Secret - -```bash -# Create temporary secret file -kubectl create secret generic smartem-secrets \ - --namespace=smartem-decisions-production \ - --from-literal=POSTGRES_USER="secure_postgres_user" \ - --from-literal=POSTGRES_PASSWORD="secure_postgres_password" \ # pragma: allowlist secret - --from-literal=RABBITMQ_USER="secure_rabbitmq_user" \ - --from-literal=RABBITMQ_PASSWORD="secure_rabbitmq_password" \ # pragma: allowlist secret - --dry-run=client \ - --output=yaml > temp-secret.yaml -``` - -### 2. Generate Sealed Secret - -```bash -# Seal the secret using cluster's public key -kubeseal --format=yaml --namespace=smartem-decisions-production < temp-secret.yaml > sealed-secret.yaml - -# Clean up temporary file -rm temp-secret.yaml -``` - -### 3. Apply Sealed Secret - -```bash -# Apply sealed secret to cluster -kubectl apply -f sealed-secret.yaml - -# Verify secret was created -kubectl get secrets -n smartem-decisions-production -``` - -## Secret Rotation - -Regular secret rotation is essential for security. Follow these steps to rotate secrets: - -### 1. Generate New Credentials - -```bash -# Use the generation script with new credentials -./tools/k8s/generate-sealed-secrets.sh production -``` - -### 2. Apply Updated Secrets - -```bash -# Apply to cluster using kustomize -kubectl apply -k k8s/environments/production/ -``` - -### 3. Restart Applications - -Restart application pods to pick up new secrets: - -```bash -# Restart SmartEM API pods -kubectl rollout restart deployment smartem-http-api -n smartem-decisions-production - -# Restart worker pods -kubectl rollout restart deployment smartem-worker -n smartem-decisions-production -``` - -### 4. Verify Application Health - -```bash -# Check pod status -kubectl get pods -n smartem-decisions-production - -# Check application logs -kubectl logs -n smartem-decisions-production deployment/smartem-http-api --tail=50 -``` - -## Integration with Kustomize - -Sealed secrets integrate seamlessly with the project's Kustomize structure: - -### Directory Structure - -``` -k8s/environments/ -├── development/ -│ ├── kustomization.yaml # References secrets.yaml -│ └── secrets.yaml # Sealed secret for development -├── staging/ -│ ├── kustomization.yaml # References secrets.yaml -│ └── secrets.yaml # Sealed secret for staging -└── production/ - ├── kustomization.yaml # References secrets.yaml - └── secrets.yaml # Sealed secret for production -``` - -### Applying Changes - -```bash -# Deploy entire environment including sealed secrets -kubectl apply -k k8s/environments/production/ - -# Apply only secrets -kubectl apply -f k8s/environments/production/secrets.yaml -``` - -## Troubleshooting - -### Common Issues - -**kubeseal command not found** -```bash -# Install kubeseal CLI tool -wget https://github.com/bitnami-labs/sealed-secrets/releases/latest/download/kubeseal-linux-amd64 -sudo install -m 755 kubeseal-linux-amd64 /usr/local/bin/kubeseal -``` - -**cannot fetch certificate error** -```bash -# Verify sealed-secrets controller is running -kubectl get pods -n kube-system -l name=sealed-secrets-controller - -# Check controller logs -kubectl logs -n kube-system -l name=sealed-secrets-controller -``` - -**secret not being decrypted** -```bash -# Check sealed secret status -kubectl describe sealedsecret smartem-secrets -n smartem-decisions - -# Verify secret was created -kubectl get secret smartem-secrets -n smartem-decisions - -# Check controller events -kubectl get events -n smartem-decisions --sort-by=.metadata.creationTimestamp -``` - -### Sealed Secret Validation - -Verify sealed secrets are correctly formatted: - -```bash -# Check sealed secret structure -kubectl get sealedsecret smartem-secrets -n smartem-decisions -o yaml - -# Validate with dry-run -kubectl apply --dry-run=client -f k8s/environments/development/secrets.yaml -``` - -### Application Connection Issues - -If applications cannot connect after secret rotation: - -```bash -# Check secret contents (base64 encoded) -kubectl get secret smartem-secrets -n smartem-decisions -o yaml - -# Test database connection -kubectl exec -it deployment/smartem-http-api -n smartem-decisions -- \ - python -c "from smartem_backend.utils import setup_postgres_connection; setup_postgres_connection()" - -# Check environment variable injection -kubectl exec -it deployment/smartem-http-api -n smartem-decisions -- env | grep POSTGRES -``` - -### Script Debugging - -For issues with the generation script: - -```bash -# Run with verbose output -bash -x ./tools/k8s/generate-sealed-secrets.sh development - -# Check prerequisites manually -kubectl cluster-info -kubeseal --version -openssl version -``` - -## Security Best Practices - -### Secret Generation - -1. **Use Strong Passwords**: Generate passwords with sufficient entropy (minimum 24 characters) -2. **Unique Credentials**: Use different credentials for each environment -3. **Regular Rotation**: Rotate secrets quarterly or after security incidents -4. **Principle of Least Privilege**: Grant minimal required database/queue permissions - -### Access Control - -1. **Namespace Isolation**: Deploy sealed secrets to appropriate namespaces -2. **RBAC Controls**: Restrict access to sealed secret resources -3. **Audit Logging**: Monitor sealed secret creation and modification -4. **Backup Strategy**: Maintain secure backups of unsealed credentials - -### Development Workflow - -1. **Never Commit Plain Secrets**: Always use sealed secrets in version control -2. **Validate Before Commit**: Verify sealed secrets are properly encrypted -3. **Environment Separation**: Keep development and production secrets separate -4. **Code Review**: Review all secret-related changes before merging - -## Advanced Usage - -### Custom Secret Names - -To use different secret names: - -```bash -# Modify script variables or create custom sealed secret -kubectl create secret generic custom-secret-name \ - --namespace=smartem-decisions \ - --from-literal=API_KEY="secure_api_key" \ # pragma: allowlist secret - --dry-run=client -o yaml | \ -kubeseal --format=yaml --namespace=smartem-decisions > custom-sealed-secret.yaml -``` - -### Multiple Secret Sources - -For complex applications requiring multiple secret sources: - -```bash -# Database secrets -./tools/k8s/generate-sealed-secrets.sh production - -# Additional API secrets -kubectl create secret generic api-secrets \ - --namespace=smartem-decisions-production \ - --from-literal=EXTERNAL_API_KEY="api_key_here" \ # pragma: allowlist secret - --dry-run=client -o yaml | \ -kubeseal --format=yaml --namespace=smartem-decisions-production > api-sealed-secrets.yaml -``` - -### Cross-Cluster Migration - -When moving between clusters, sealed secrets must be regenerated: - -```bash -# Export existing secret from source cluster -kubectl get secret smartem-secrets -n smartem-decisions -o yaml > plain-secret.yaml - -# Remove cluster-specific metadata -# Re-seal for target cluster -kubeseal --format=yaml --namespace=smartem-decisions < plain-secret.yaml > new-sealed-secret.yaml - -# Clean up plain secret file -rm plain-secret.yaml -``` - -## Files and Structure - -``` -├── tools/k8s/generate-sealed-secrets.sh # Main secret generation script -├── k8s/environments/ -│ ├── development/ -│ │ ├── secrets.yaml # Development sealed secrets -│ │ └── kustomization.yaml # References secrets.yaml -│ ├── staging/ -│ │ ├── secrets.yaml # Staging sealed secrets -│ │ └── kustomization.yaml # References secrets.yaml -│ └── production/ -│ ├── secrets.yaml # Production sealed secrets -│ └── kustomization.yaml # References secrets.yaml -└── k8s/secret.example.yaml # Example plain secret structure -``` - -## Further Reading - -- [Bitnami Sealed Secrets Documentation](https://sealed-secrets.netlify.app/) -- [Kubernetes Secrets Documentation](https://kubernetes.io/docs/concepts/configuration/secret/) -- [Diamond Light Source Kubernetes Standards](https://confluence.diamond.ac.uk/display/DASCM/Kubernetes) -- [Security Best Practices for Kubernetes](https://kubernetes.io/docs/concepts/security/) diff --git a/webui/src/docs/how-to/run-agent.mdx b/webui/src/docs/how-to/run-agent.mdx deleted file mode 100644 index 6c5c571..0000000 --- a/webui/src/docs/how-to/run-agent.mdx +++ /dev/null @@ -1,329 +0,0 @@ -# Run SmartEM Agent (EPU Agent) - -The SmartEM Agent is a data collection service that monitors EPU (Electron Physical User) output directories and communicates acquisition data to the backend service in real-time. - -## Overview - -The EPU agent runs on EPU workstations either as a Python script or bundled Windows binary. EPU workstations are typically Windows machines isolated from the main network, where specific connectivity is achieved through a proxy and configured via an allow-list. The primary purpose of the EPU agent is to parse EPU software output from the filesystem and communicate data and events to the core backend component. - -An EPU data directory is generated by closed-source EPU software and represents an acquisition session using a cryo-electron microscope. The SmartEM Agent provides comprehensive capabilities for processing this data: - -### Core Capabilities - -- **Real-time monitoring**: Watch EPU directories for file changes during active acquisitions -- **Comprehensive parsing**: Extract data from all EPU file types (sessions, atlases, grid squares, foil holes, micrographs) -- **Data validation**: Verify EPU directory structure and completeness -- **Backend integration**: Communicate with SmartEM backend via REST API and Server-Sent Events (SSE) -- **Connection health**: Automatic heartbeat monitoring for reliable data transmission -- **Flexible deployment**: Run in development, testing, or production modes - -### Agent Modes - -The agent operates in several modes depending on the timing of EPU data acquisition: - -1. **Pre-acquisition mode**: Watcher launched _before_ EPU starts writing - real-time monitoring only -2. **Mid-acquisition mode**: Watcher launched _after_ EPU starts writing - combines parsing existing files with real-time monitoring -3. **Post-acquisition mode**: Watcher launched _after_ EPU finishes - parses complete dataset then monitors for changes - -## Quick Start - -For comprehensive parameter documentation, see the [CLI Reference](../reference/cli-reference.md). For troubleshooting, see the [CLI Troubleshooting Guide](troubleshoot-cli.md). - -### Basic Directory Monitoring - -```bash -# Monitor a directory with default settings -python -m smartem_agent watch /path/to/epu/data - -# Monitor with verbose output -python -m smartem_agent watch /path/to/epu/data --verbose - -# Dry run for testing (no API calls) -python -m smartem_agent watch /path/to/epu/data --dry-run --verbose -``` - -### Production Deployment with Backend Integration - -```bash -# Full production setup with real-time communication -python -m smartem_agent watch /data/microscopy/active_session \ - --api-url https://smartem-backend.facility.ac.uk \ - --agent-id microscope-titan-01 \ - --session-id session-20240115-001 \ - --heartbeat-interval 45 \ - --verbose -``` - -## Command Categories - -The SmartEM Agent CLI is organised into three main command categories: - -### 1. Parse Commands -Extract and analyse data from EPU files without backend communication. Useful for development, debugging, and data validation. - -### 2. Validate Commands -Check EPU directory structure for completeness and compliance with expected formats. - -### 3. Watch Commands -Monitor directories in real-time for file changes with full backend integration. - -## Parsing Operations - -Parse commands extract and analyse data from EPU files without communicating with the backend API. These commands are ideal for development, debugging, data validation, and understanding EPU data structures. - -### Complete Directory Parsing - -Parse entire EPU directories containing multiple grids or complete acquisition sessions: - -```bash -# Parse complete EPU session directory -python -m smartem_agent parse dir \ - ../smartem-decisions-test-datasets/metadata_Supervisor_20250108_101446_62_cm40593-1_EPU - -# Parse different session types -python -m smartem_agent parse dir \ - ../smartem-decisions-test-datasets/metadata_Supervisor_20250114_220855_23_epuBSAd20_GrOxDDM \ - --verbose - -python -m smartem_agent parse dir \ - ../smartem-decisions-test-datasets/metadata_Supervisor_20241220_140307_72_et2_gangshun \ - --verbose --verbose # Debug level output -``` - -### Individual Component Parsing - -Parse specific EPU file types to understand data structures and debug issues: - -#### Session Files -```bash -# Parse EPU session manifest -python -m smartem_agent parse session \ - ../smartem-decisions-test-datasets/bi37708-28-copy/Supervisor_20250129_134723_36_bi37708-28_grid7_EPU/EpuSession.dm \ - --verbose -``` - -#### Atlas Files -```bash -# Parse atlas overview data -python -m smartem_agent parse atlas \ - ../smartem-decisions-test-datasets/bi37708-28-copy/atlas/Supervisor_20250129_111544_bi37708-28_atlas/Atlas/Atlas.dm \ - --verbose -``` - -#### Grid Square Files -```bash -# Parse grid square manifest (XML format) -python -m smartem_agent parse gridsquare \ - ../smartem-decisions-test-datasets/epu-Supervisor_20250404_164354_31_EPU_nr27313-442/metadata_Supervisor_20250404_164354_31_EPU_nr27313-442/Images-Disc1/GridSquare_3568837/GridSquare_20250404_171012.xml - -# Parse grid square metadata (DM format) -python -m smartem_agent parse gridsquare-metadata \ - ../smartem-decisions-test-datasets/epu-Supervisor_20250404_164354_31_EPU_nr27313-442/metadata_Supervisor_20250404_164354_31_EPU_nr27313-442/Metadata/GridSquare_3568837.dm \ - --verbose - -# Alternative dataset example -python -m smartem_agent parse gridsquare-metadata \ - ./tests/testdata/bi37708-28/Supervisor_20250129_134723_36_bi37708-28_grid7_EPU/Metadata/GridSquare_29273435.dm -``` - -#### Foil Hole Files -```bash -# Parse foil hole positioning data -python -m smartem_agent parse foilhole \ - tests/testdata/epu-dir-example/Images-Disc1/GridSquare_8999138/FoilHoles/FoilHole_9015889_20250108_154725.xml \ - --verbose - -# Parse foil hole acquisition data (alternative location) -python -m smartem_agent parse micrograph \ - ../smartem-decisions-test-datasets/epu-Supervisor_20250404_164354_31_EPU_nr27313-442/metadata_Supervisor_20250404_164354_31_EPU_nr27313-442/Images-Disc1/GridSquare_3568837/Data/FoilHole_3595930_Data_3590445_56_20250405_084025.xml -``` - -## Validation Operations - -Validation commands check EPU directory structure for completeness and compliance with expected formats. - -### Examples with Expected Outcomes - -#### Invalid Directories (Expected to Fail) -These examples demonstrate directories with structural issues: - -```bash -# Incomplete or malformed directories -python -m smartem_agent validate \ - ../smartem-decisions-test-datasets/bi37708-28-copy/Supervisor_20250129_114842_73_bi37708-28_grid7_EPU \ - --verbose - -python -m smartem_agent validate \ - ../smartem-decisions-test-datasets/bi37708-28-copy/Supervisor_20250130_105058_11 \ - --verbose - -python -m smartem_agent validate \ - ../smartem-decisions-test-datasets/bi37708-28-copy/Supervisor_20250130_145409_68 - -python -m smartem_agent validate \ - ../smartem-decisions-test-datasets/bi37708-28-copy/Supervisor_20250130_150924_1grid3 -``` - -#### Valid Directories (Expected to Pass) -These examples show properly structured EPU directories: - -```bash -# Complete, well-formed directories -python -m smartem_agent validate \ - ../smartem-decisions-test-datasets/bi37708-28-copy/Supervisor_20250129_134723_36_bi37708-28_grid7_EPU \ - --verbose - -python -m smartem_agent validate \ - ../smartem-decisions-test-datasets/bi37708-28-copy/Supervisor_20250130_133418_68apoferritin \ - --verbose - -python -m smartem_agent validate \ - ../smartem-decisions-test-datasets/bi37708-28-copy/Supervisor_20250130_143856_44Practice - -python -m smartem_agent validate \ - ../smartem-decisions-test-datasets/bi37708-28-copy/Supervisor_20250130_145409_68practice2 -``` - -### Understanding Validation Results - -**Successful validation** returns exit code 0 and confirms the directory structure is valid: -``` -EPU project dir is structurally valid -``` - -**Failed validation** returns exit code 1 and lists specific issues: -``` -Invalid EPU project dir. Found the following issues: -- Missing required file: EpuSession.dm -- Invalid directory structure: Images-Disc1 not found -- Incomplete atlas data: Atlas/Atlas.dm missing -``` - -## Real-Time Monitoring (Watch Operations) - -The watch command provides real-time monitoring of EPU directories, automatically processing new files and communicating with the SmartEM backend. This is the primary operational mode for production deployments. - -### Basic Watch Operations - -```bash -# Simple directory monitoring (development mode) -python -m smartem_agent watch ../test-dir --log-file output.log - -# Monitor with detailed logging -python -m smartem_agent watch /data/microscopy/active_session \ - --log-file /var/log/smartem/session.log \ - --log-interval 5.0 \ - --verbose - -# Dry run for testing (no backend communication) -python -m smartem_agent watch ../test-dir \ - --dry-run \ - --verbose --verbose -``` - -### Production Monitoring with Backend Integration - -```bash -# Full production deployment -python -m smartem_agent watch /data/microscopy/active_session \ - --api-url https://smartem-backend.facility.ac.uk \ - --agent-id microscope-titan-01 \ - --session-id session-20240115-001 \ - --heartbeat-interval 45 \ - --sse-timeout 60 \ - --log-interval 10.0 \ - --verbose - -# High-frequency monitoring setup -python -m smartem_agent watch /data/high_throughput \ - --api-url http://backend:8000 \ - --agent-id facility-workstation-03 \ - --session-id batch-processing-session \ - --heartbeat-interval 30 \ - --log-interval 5.0 \ - --sse-timeout 120 -``` - -### Watch Operation Modes - -The watch command is designed to gracefully handle different timing scenarios relative to EPU data acquisition: - -1. **Pre-acquisition mode**: Watcher launched _before_ EPU starts writing - - Only real-time monitoring is necessary - - Files are processed as they are created - - Most efficient mode for active acquisitions - -2. **Mid-acquisition mode**: Watcher launched _after_ EPU starts writing - - Combines initial parsing of existing files with real-time monitoring - - Automatically detects and processes pre-existing data - - Seamlessly transitions to real-time monitoring - -3. **Post-acquisition mode**: Watcher launched _after_ EPU finishes - - Parses complete dataset then monitors for any changes - - Useful for processing archived or completed datasets - - Continues monitoring for potential updates - -### Testing with Simulated EPU Data - -For development and testing, use the fsrecorder tool to simulate realistic EPU file writing patterns: - -#### Recording EPU Patterns -```bash -# Record filesystem events from an existing EPU dataset -python tools/fsrecorder/fsrecorder.py record \ - ../smartem-decisions-test-datasets/epu-Supervisor_20250326_145351_30_nt33824-10_grid2_1in5dil \ - ../test-recording.tar.gz -``` - -#### Replaying EPU Patterns -```bash -# Replay recorded events with accelerated timing -python tools/fsrecorder/fsrecorder.py replay \ - ../test-recording.tar.gz \ - ../test-dir \ - --fast - -# Monitor the replayed data -python -m smartem_agent watch ../test-dir \ - --dry-run \ - --verbose --verbose \ - --log-interval 2.0 -``` - -#### Quick Testing Alternative -```bash -# For rapid testing, copy data manually (less realistic timing) -cp -r "../smartem-decisions-test-datasets/epu-Supervisor_20250326_145351_30_nt33824-10_grid2_1in5dil/"* ../test-dir/ - -# Monitor the copied data -python -m smartem_agent watch ../test-dir --dry-run -``` - -### Important Considerations - -> **Critical File**: The `EpuSession.dm` file is essential for proper operation as it: -> - Provides references to atlas data -> - Triggers instantiation of new grid entities in the datastore -> - Contains acquisition metadata required for processing -> -> **Missing `EpuSession.dm`**: Will prevent proper grid instantiation and data processing. - -> **fsrecorder Tool**: The `tools/fsrecorder/` utility provides accurate simulation of EPU file writing patterns with proper timing and ordering, making it ideal for development and testing scenarios. - -### Real-Time Communication Features - -When using `--agent-id` and `--session-id` parameters, the agent establishes real-time communication with the backend: - -- **Server-Sent Events (SSE)**: Receives instructions and commands from the backend -- **Heartbeat Monitoring**: Sends periodic heartbeats to maintain connection health -- **Automatic Reconnection**: Handles connection failures with exponential backoff -- **Instruction Acknowledgment**: Confirms receipt and processing of backend instructions - -### Performance Tuning - -Adjust parameters based on your deployment requirements: - -- **High-frequency acquisitions**: Lower `--log-interval` (1-5 seconds), lower `--heartbeat-interval` (15-30 seconds) -- **Stable networks**: Standard `--sse-timeout` (30-60 seconds) -- **Unstable networks**: Higher `--sse-timeout` (120+ seconds), higher `--heartbeat-interval` (60+ seconds) -- **Development/testing**: Use `--dry-run` to avoid backend communication diff --git a/webui/src/docs/how-to/run-backend.mdx b/webui/src/docs/how-to/run-backend.mdx deleted file mode 100644 index 5a1223d..0000000 --- a/webui/src/docs/how-to/run-backend.mdx +++ /dev/null @@ -1,39 +0,0 @@ -# Run SmartEM Backend - -The core backend service providing HTTP API, database operations, and message queue processing for intelligent cryo-EM data collection. - -## Backend Operations - -```bash -# create env and launch service stack locally: -./tools/k8s/dev-k8s.sh up - -# launch RabbitMQ worker (consumer) -python -m smartem_backend.consumer # ERROR level (default) -python -m smartem_backend.consumer -v # INFO level -python -m smartem_backend.consumer -vv # DEBUG level - -# simulating an system event: -python -m smartem_backend.simulate_msg --help # to see a list of options -./tools/simulate-messages.sh # run a simulation, triggering system events in sequence - -# run HTTP API (recommended - uses proper module entry point): -python -m smartem_backend.api_server - -# run HTTP API in development with FastAPI CLI: -fastapi dev src/smartem_backend/api_server.py # Note: FastAPI CLI gets installed by pip as one of dev dependencies - -# run HTTP API with uvicorn directly: -source .env && uvicorn smartem_backend.api_server:app --host 0.0.0.0 --port $HTTP_API_PORT - -# run HTTP API with environment variable: -SMARTEM_LOG_LEVEL=ERROR uvicorn smartem_backend.api_server:app --host 0.0.0.0 --port $HTTP_API_PORT -# customize HTTP API host/port with environment variables: -HTTP_API_PORT=9000 python -m smartem_backend.api_server -HTTP_API_HOST=0.0.0.0 HTTP_API_PORT=8080 python -m smartem_backend.api_server - -# smoke test the API: -./tests/check_smartem_core_http_api.py http://localhost:8000 -v - -python -m smartem_backend --version -``` diff --git a/webui/src/docs/how-to/run-container.mdx b/webui/src/docs/how-to/run-container.mdx deleted file mode 100644 index 058ec84..0000000 --- a/webui/src/docs/how-to/run-container.mdx +++ /dev/null @@ -1,72 +0,0 @@ -# Run in a container - -Pre-built containers with smartem-decisions and its dependencies already -installed are available on [Github Container Registry](https://ghcr.io/DiamondLightSource/smartem-decisions). - -## Starting the container - -To pull the container from github container registry and run: - -```bash -# Check version -$ docker run ghcr.io/DiamondLightSource/smartem-decisions:latest --version -``` - -To get a released version, use a numbered release instead of `latest`. - -## Running services - -The container can run in different modes controlled by the `ROLE` environment variable: - -### HTTP API Service - -```bash -# Run API with default ERROR logging -$ docker run -p 8000:8000 -e ROLE=api ghcr.io/DiamondLightSource/smartem-decisions:latest - -# Run API with INFO logging (equivalent to -v flag) -$ docker run -p 8000:8000 -e ROLE=api -e SMARTEM_LOG_LEVEL=INFO ghcr.io/DiamondLightSource/smartem-decisions:latest - -# Run API with DEBUG logging (equivalent to -vv flag) -$ docker run -p 8000:8000 -e ROLE=api -e SMARTEM_LOG_LEVEL=DEBUG ghcr.io/DiamondLightSource/smartem-decisions:latest - -# Custom port -$ docker run -p 9000:9000 -e ROLE=api -e HTTP_API_PORT=9000 ghcr.io/DiamondLightSource/smartem-decisions:latest -``` - -### Message Queue Worker - -```bash -# Run worker with default ERROR logging -$ docker run -e ROLE=worker ghcr.io/DiamondLightSource/smartem-decisions:latest - -# Run worker with INFO logging -$ docker run -e ROLE=worker -e SMARTEM_LOG_LEVEL=INFO ghcr.io/DiamondLightSource/smartem-decisions:latest - -# Run worker with DEBUG logging -$ docker run -e ROLE=worker -e SMARTEM_LOG_LEVEL=DEBUG ghcr.io/DiamondLightSource/smartem-decisions:latest -``` - -## Environment Variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `ROLE` | `api` | Service role: `api` or `worker` | -| `HTTP_API_PORT` | `8000` | Port for HTTP API service | -| `SMARTEM_LOG_LEVEL` | `ERROR` | Log level: `ERROR`, `INFO`, or `DEBUG` | - -## Complete Development Stack - -For a complete development environment with database and message queue, see the [Kubernetes deployment guide](../../k8s/README.md) which provides a docker-compose-like experience. - -## Advanced Configuration - -### Custom User/Group for Production Deployments - -The pre-built containers run as root by default, which is suitable for development and CI/CD. For production deployments at Diamond Light Source that require access to the `/dls` filesystem, you'll need to build custom images with specific UID/GID. - -See [Container User Configuration](container-user-configuration.md) for: -- Building containers with custom user/group settings -- Mounting the `/dls` filesystem for image serving -- Security considerations and best practices -- Troubleshooting permission issues diff --git a/webui/src/docs/how-to/run-e2e-dev-simulation.mdx b/webui/src/docs/how-to/run-e2e-dev-simulation.mdx deleted file mode 100644 index 4805ea9..0000000 --- a/webui/src/docs/how-to/run-e2e-dev-simulation.mdx +++ /dev/null @@ -1,828 +0,0 @@ -# End-to-End Development Test Runs - -This document describes the complete workflow for running repeatable end-to-end tests of the SmartEM system using pre-recorded microscope sessions. It serves as both a runbook for executing tests and a reference for understanding test requirements. - -## Quick Start (Automated Test Runner) - -For the simplest test execution, use the automated test runner script: - -```bash -# Check k3s status first (DO NOT restart if already running) -./tools/k8s/dev-k8s.sh status - -# Run automated test (Test Type 2: Pre-Acquisition Agent Setup) -./tools/run-e2e-test.sh - -# With custom parameters (paths relative to workspace root) -./tools/run-e2e-test.sh \ - ../../testdata/recordings/bi37708-42_fsrecord.tar.gz \ - ../../tmp/epu-test-dir \ - 0.1 - -# Or override via environment variables -SMARTEM_TEST_RECORDING=/path/to/recording.tar.gz \ -SMARTEM_EPU_DIR=/path/to/epu-dir \ -./tools/run-e2e-test.sh -``` - -**Script parameters** (all optional): -1. Recording file path (default: `/testdata/recordings/bi37708-42_fsrecord.tar.gz`) -2. EPU output directory (default: `/tmp/epu-test-dir`) -3. Max delay in seconds (default: `0.1`) - -**Environment variable overrides**: -- `SMARTEM_TEST_RECORDING`: Override default recording path -- `SMARTEM_EPU_DIR`: Override default EPU directory - -**Workspace structure**: Scripts assume a multi-repo workspace layout where this repo lives at `/DiamondLightSource/smartem-decisions/`. - -**What the script does**: -- Creates timestamped test results directory in `logs/e2e-tests/` -- Activates venv and loads environment variables from `.env.local-test-run` -- Resets database to clean state -- Starts API server and consumer -- Starts agent watching empty directory -- Runs playback with specified timing -- Collects filesystem and database statistics -- Saves all logs to test results directory -- Cleans up background processes on exit - -**When to use**: -- Quick smoke tests during development -- Consistent repeatable test runs -- Pre-Acquisition scenario testing (Test Type 2) - -**When to use manual execution instead**: -- Testing Post-Acquisition scenario (Test Type 1) -- Testing Mid-Acquisition scenario (Test Type 3) -- Debugging specific service interactions -- Custom timing or service startup order - -## Quick Start (Manual Execution) - -If you've already set up once and just need to run another test: - -```bash -# 1. Check k3s status (DO NOT restart if already running) -./tools/k8s/dev-k8s.sh status - -# 2. Prepare clean environment -unset POSTGRES_HOST POSTGRES_PORT POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD -unset RABBITMQ_HOST RABBITMQ_PORT RABBITMQ_USER RABBITMQ_PASSWORD -unset HTTP_API_HOST HTTP_API_PORT -set -a && source .env && set +a - -# 3. Create timestamped test directory -TEST_DIR="logs/e2e-tests/$(date +%Y-%m-%d_%H%M%S)_test-type-name" -mkdir -p "$TEST_DIR/logs" - -# 4. Kill any running processes -pkill -f "smartem_backend\|smartem_agent\|fsrecorder\|uvicorn\|fastapi" || true - -# 5. Restart RabbitMQ (ensures empty queue) -kubectl rollout restart deployment/rabbitmq -n smartem-decisions -kubectl rollout status deployment/rabbitmq -n smartem-decisions - -# 6. Reset database -source .venv/bin/activate -python -m smartem_backend.model.database - -# 7. Clean playback directory -rm -rf ../epu-test-dir && mkdir -p ../epu-test-dir - -# 8. Start services (choose a test type below, or see Test Execution Scenarios section) -# Example: Test Type 1 - Post-acquisition -python -m uvicorn smartem_backend.api_server:app --host 127.0.0.1 --port 8000 --log-level debug 2>&1 | tee "$TEST_DIR/logs/api.log" & -python -m smartem_backend.consumer -vv 2>&1 | tee "$TEST_DIR/logs/consumer.log" & -sleep 3 - -# Run playback (default is --fast mode, no flag needed) -python ./tools/fsrecorder/fsrecorder.py replay \ - ../../testdata/recordings/bi37708-42_fsrecord.tar.gz \ - ../epu-test-dir 2>&1 | tee "$TEST_DIR/logs/playback.log" - -# Start agent after playback completes -python -m smartem_agent watch --api-url http://localhost:8000 -vv ../epu-test-dir 2>&1 | tee "$TEST_DIR/logs/agent.log" -``` - -**For detailed explanations, first-time setup, or different test scenarios, see sections below.** - ---- - -## Multi-Microscope Test (Concurrent Sessions) - -For testing multiple concurrent microscopes and acquisition sessions simultaneously, use the multi-microscope test runner: - -```bash -# Check k3s status first (DO NOT restart if already running) -./tools/k8s/dev-k8s.sh status - -# Run multi-microscope test with 3 microscopes (default) -./tools/run-e2e-test-multi-microscope.sh - -# With custom parameters (paths relative to workspace root) -./tools/run-e2e-test-multi-microscope.sh \ - 3 \ - ../../testdata/recordings/bi37708-42_fsrecord.tar.gz \ - ../../tmp/epu-test-dir \ - 0.1 - -# Or override via environment variables -SMARTEM_TEST_RECORDING=/path/to/recording.tar.gz \ -SMARTEM_EPU_DIR=/path/to/epu-dir \ -./tools/run-e2e-test-multi-microscope.sh 3 -``` - -**Script parameters** (all optional): -1. Number of microscopes (default: `3`) -2. Recording file path (default: `/testdata/recordings/bi37708-42_fsrecord.tar.gz`) -3. EPU base directory (default: `/tmp/epu-test-dir`) - - Each microscope gets a separate directory: `epu-test-dir-microscope-1`, `epu-test-dir-microscope-2`, etc. -4. Max delay in seconds (default: `0.1`) - -**Environment variable overrides**: -- `SMARTEM_TEST_RECORDING`: Override default recording path -- `SMARTEM_EPU_DIR`: Override default EPU directory - -**Workspace structure**: Scripts assume a multi-repo workspace layout where this repo lives at `/DiamondLightSource/smartem-decisions/`. - -**What the script does**: -- Creates timestamped test results directory in `logs/e2e-tests/` -- Activates venv and loads environment variables from `.env.local-test-run` -- Resets database to clean state -- Starts single API server and consumer (shared by all microscopes) -- Starts N agent instances, each with unique: - - Agent ID (e.g., `microscope-titan-01`, `microscope-titan-02`, `microscope-titan-03`) - - Session ID (e.g., `session-20250110-001`, `session-20250110-002`, `session-20250110-003`) - - EPU directory for playback isolation -- Runs N concurrent playback instances to separate directories -- Collects and verifies: - - Filesystem statistics per microscope - - Database counts per acquisition - - Agent session associations - - Data separation between acquisitions -- Saves all logs to test results directory (N agent logs, N playback logs, 1 API log, 1 consumer log) -- Cleans up background processes and directories on exit - -**When to use**: -- Testing multi-microscope facility scenarios -- Validating data isolation in the database -- Testing SSE routing correctness (each agent receives only its own instructions) -- Identifying race conditions and resource contention -- Testing concurrent agent/backend operations -- Verifying no data leakage between acquisitions - -**Success criteria**: -- Each agent creates a separate acquisition in the database -- All acquisitions have completely separate data (no shared grids/gridsquares/foilholes) -- Each agent receives SSE instructions only for its own session -- Database counts match filesystem counts for each microscope -- No errors in any service logs -- Data separation verification passes - -**Example output**: -``` -===== Test Results ===== - -Filesystem Counts: - Microscope 1 (microscope-titan-01): - EPU Sessions: 1 - GridSquare directories: 42 - Microscope 2 (microscope-titan-02): - EPU Sessions: 1 - GridSquare directories: 42 - Microscope 3 (microscope-titan-03): - EPU Sessions: 1 - GridSquare directories: 42 - -Database Counts (per acquisition): - Total acquisitions: 3 - - Acquisition 1 (TK-01): - UUID: abc-123... - Grids: 1 - Grid Squares: 42 - Foil Holes: 156 - - Acquisition 2 (TK-02): - UUID: def-456... - Grids: 1 - Grid Squares: 42 - Foil Holes: 156 - - Acquisition 3 (TK-03): - UUID: ghi-789... - Grids: 1 - Grid Squares: 42 - Foil Holes: 156 - -===== Data Separation Verification ===== - - Found 3 acquisitions. - - SUCCESS: All acquisitions have completely separate data. - No grid UUID overlaps detected between acquisitions. -``` - ---- - -## Overview - -The test setup simulates a complete SmartEM workflow: -- **Local k3s cluster**: Runs PostgreSQL and RabbitMQ services -- **Host OS services**: Runs SmartEM backend API and worker for easier debugging -- **Microscope simulation**: Plays back pre-recorded microscopy sessions -- **Agent monitoring**: SmartEM agent watches for file changes and processes data - -## Prerequisites - -### Environment Setup -- Python 3.12+ with venv activated: `source .venv/bin/activate` -- Full development install: `pip install -e .[all]` -- Local k3s cluster running: `./tools/k8s/dev-k8s.sh up` -- Environment file: `.env` (created from `.env.example` - see Environment File Setup below) - -### Test Data -Pre-recorded microscope sessions are stored in `/testdata/recordings/`: -- `bi37600-29_fsrecord.tar.gz` -- `bi37708-42_fsrecord.tar.gz` (recommended for testing, 8389 events) - -**Note**: Scripts auto-detect workspace structure. Override with `SMARTEM_TEST_RECORDING` env var if needed. - -### Directory Structure -``` -logs/e2e-tests/ # Test results root (gitignored, at repo root) - YYYY-MM-DD_HHmmss_test-type/ # Individual test run - logs/ # All log outputs (4 services) - playback.log # Microscope recording playback - agent.log # SmartEM agent - api.log # Backend HTTP API - consumer.log # Backend worker/consumer - db-dump.sql # Database state after test - test-params.json # Test configuration used -``` - -### Environment File Setup - -The `.env.local-test-run` file configures services to run on host OS whilst connecting to k3s infrastructure. - -**Source for initial values**: Copy from `.env.example.local-test-run` and configure credentials - -**Required configuration**: -- PostgreSQL: Point to k3s NodePort `localhost:30432` -- RabbitMQ: Point to k3s NodePort `localhost:30672` -- Backend API: Run on host OS (typically `127.0.0.1:8000`) -- Agent: Run on host OS - -**Example `.env.local-test-run`**: -```bash -# Database configuration (connecting to K8s NodePort) -POSTGRES_HOST=localhost -POSTGRES_PORT=30432 -POSTGRES_DB=smartem_db -POSTGRES_USER=username -POSTGRES_PASSWORD=password - -# RabbitMQ configuration (connecting to K8s NodePort) -RABBITMQ_HOST=localhost -RABBITMQ_PORT=30672 -RABBITMQ_UI_PORT=15672 -RABBITMQ_USER=username -RABBITMQ_PASSWORD=password - -# HTTP API configuration (running on host OS) -HTTP_API_PORT=8000 -ADMINER_PORT=8080 - -# CORS Configuration -CORS_ALLOWED_ORIGINS=* -``` - -**Usage**: Automatically loaded by `./tools/run-e2e-test.sh` script - -## Database Operations - -### Creating Fresh Database -Run `database.py` to drop all tables and recreate schema with indexes: -```bash -source .venv/bin/activate -python -m smartem_backend.model.database -``` - -**Note**: This script uses `_create_db_and_tables()` which: -1. Drops all existing tables and custom enum types -2. Creates all tables from SQLModel definitions -3. Creates all indexes for optimal query performance -4. Does NOT require Alembic migrations - -### Connecting to Database -From host OS (using k3s NodePort mappings): -```bash -# Connection details (from .env) -POSTGRES_HOST=localhost -POSTGRES_PORT=30432 -POSTGRES_DB=smartem_db -POSTGRES_USER=username -POSTGRES_PASSWORD=password - -# Example: Dump database for test results -pg_dump -h localhost -p 30432 -U username -d smartem_db > logs/e2e-tests/TIMESTAMP/db-dump.sql -``` - -## RabbitMQ Queue Management - -### Queue Behaviour -- Queue name: `smartem_backend` (configured in `src/smartem_backend/appconfig.yml`) -- Queue is declared with `durable=True` by both publisher and consumer -- Queue persists across reconnections but NOT across RabbitMQ service restarts -- **Important**: Queue is NOT recreated empty on each connection - existing messages persist - -### Verifying Queue State -Access RabbitMQ management UI: -```bash -# RabbitMQ management interface (if enabled in k3s) -# Check k8s service for management UI port mapping -# Default credentials: guest/guest # pragma: allowlist secret -``` - -### Cleaning Queue Between Tests -**Option 1**: Restart RabbitMQ service (recommended for clean state): -```bash -kubectl rollout restart deployment/rabbitmq -n smartem-decisions -kubectl rollout status deployment/rabbitmq -n smartem-decisions -``` - -**Option 2**: Purge queue programmatically (if needed): -```python -# Add to teardown script if queue purging is required -# This would need to be implemented in utils.py -``` - -## Service Commands - -**IMPORTANT - Environment Variables**: -All services use `load_dotenv(override=False)` which loads `.env` by default but doesn't override already-set variables. -For testing, you MUST export variables from `.env` before starting services: - -```bash -# Unset any previously loaded env vars to avoid conflicts -unset POSTGRES_HOST POSTGRES_PORT POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD -unset RABBITMQ_HOST RABBITMQ_PORT RABBITMQ_USER RABBITMQ_PASSWORD -unset HTTP_API_HOST HTTP_API_PORT - -# Load test environment (exports all variables) -set -a -source .env -set +a -``` - -### Backend HTTP API - -**RECOMMENDED: Uvicorn without auto-reload (for e2e tests)** -```bash -source .venv/bin/activate -python -m uvicorn smartem_backend.api_server:app \ - --host 127.0.0.1 \ - --port 8000 \ - --log-level debug 2>&1 | tee logs/e2e-tests/TIMESTAMP/logs/api.log -``` -- **Log level options**: `critical`, `error`, `warning`, `info`, `debug`, `trace` -- No auto-reload to avoid file watch limits (important for repos with .tox directories) -- More verbose logging control -- Stable for long-running tests - -**Alternative: FastAPI CLI (NOT recommended for e2e tests)** -```bash -source .venv/bin/activate -python -m fastapi dev src/smartem_backend/api_server.py \ - --host 127.0.0.1 \ - --port 8000 2>&1 | tee /tmp/e2e-test-results/TIMESTAMP/logs/api.log -``` -- Auto-reload on code changes (watches entire project directory) -- **WARNING**: Will fail with "OS file watch limit reached" error if .tox or other large directories exist -- Only use for quick manual development, not for e2e test runs - -**Note**: Both read database and RabbitMQ connection details from environment variables - -### Backend Worker/Consumer -```bash -source .venv/bin/activate -export RABBITMQ_URL=amqp://guest:guest@localhost:30672/ # pragma: allowlist secret -python -m smartem_backend.consumer -vv 2>&1 | tee logs/e2e-tests/TIMESTAMP/logs/consumer.log -``` -**Verbosity flags**: `-v` for INFO, `-vv` for DEBUG - -### SmartEM Agent - -```bash -source .venv/bin/activate -python -m smartem_agent watch \ - --api-url http://localhost:30080 \ - --verbose -v \ - ../epu-test-dir 2>&1 | tee logs/e2e-tests/TIMESTAMP/logs/agent.log -``` - -**Verbosity flags**: `-v` for INFO, `-vv` for DEBUG - -**Agent options**: -- `--agent-id TEXT`: Agent identifier for SSE connection -- `--session-id TEXT`: Session identifier for SSE connection -- `--sse-timeout INTEGER`: SSE connection timeout in seconds (default: 30) -- `--heartbeat-interval INTEGER`: Agent heartbeat interval in seconds (default: 60) - -### Microscope Playback (fsrecorder) - -**Tool location**: `./tools/fsrecorder/fsrecorder.py` (relative to repo root) - -**Command syntax**: -```bash -source .venv/bin/activate -python ./tools/fsrecorder/fsrecorder.py replay \ - ../../testdata/recordings/bi37708-42_fsrecord.tar.gz \ - ../epu-test-dir 2>&1 | tee logs/e2e-tests/TIMESTAMP/logs/playback.log -``` - -**Important**: -- Use `replay` subcommand (NOT `--mode replay` or `--mode player`) -- First positional argument is the recording file (`.tar.gz`, not `.events`) -- Second positional argument is the target directory (no trailing slash needed) -- Recording file path can be absolute or relative to current directory (e.g., `../../testdata/recordings/...`) - -**Speed control options** (from `--help`): -- Default behavior: `--fast` mode (100x speed, 1s max delays) - **enabled by default** -- `--dev-mode`: Maximum speed for rapid iteration (1000x + burst) - quick smoke tests -- `--fast`: Balanced acceleration (100x + 1s delays) - same as default -- `--exact`: Preserve original timing (1x speed) - debug timing issues -- `-s SPEED, --speed SPEED`: Custom speed multiplier -- `--max-delay MAX_DELAY`: Maximum delay between events in seconds -- `--burst`: Process events as fast as possible - -**Additional options**: -- `--no-verify`: Skip integrity verification -- `--skip-unreadable`: Skip files that were unreadable during recording - -**Example usage**: -```bash -# Default fast mode (no flag needed) -python ./tools/fsrecorder/fsrecorder.py replay recording.tar.gz ../output-dir - -# Dev mode for rapid testing -python ./tools/fsrecorder/fsrecorder.py replay --dev-mode recording.tar.gz ../output-dir - -# Exact timing for debugging -python ./tools/fsrecorder/fsrecorder.py replay --exact recording.tar.gz ../output-dir -``` - -## Test Preparation Workflow - -### 1. Backup Previous Test Logs -```bash -# If previous test exists, logs are already in logs/e2e-tests/PREVIOUS_TIMESTAMP/ -# No action needed - each test creates its own timestamped directory -``` - -### 2. Prepare Test Environment -```bash -# 1. Unset any previously loaded environment variables -unset POSTGRES_HOST POSTGRES_PORT POSTGRES_DB POSTGRES_USER POSTGRES_PASSWORD -unset RABBITMQ_HOST RABBITMQ_PORT RABBITMQ_USER RABBITMQ_PASSWORD -unset HTTP_API_HOST HTTP_API_PORT - -# 2. Load test environment variables (export all) -set -a -source .env -set +a - -# 3. Create test results directory with timestamp -TEST_DIR="logs/e2e-tests/$(date +%Y-%m-%d_%H%M%S)_test-type-name" -mkdir -p "$TEST_DIR/logs" - -# 4. Stop any running services from previous test -pkill -f smartem_backend.api -pkill -f smartem_backend.consumer -pkill -f smartem_agent -pkill -f fsrecorder -pkill -f uvicorn -pkill -f fastapi - -# 5. Verify k3s services are running (DO NOT restart if already running) -./tools/k8s/dev-k8s.sh status - -# 6. Clean RabbitMQ (restart to ensure empty queue) -kubectl rollout restart deployment/rabbitmq -n smartem-decisions -kubectl rollout status deployment/rabbitmq -n smartem-decisions - -# 7. Drop and recreate database -source .venv/bin/activate -python -m smartem_backend.model.database - -# 8. Clean playback output directory -rm -rf ../epu-test-dir -mkdir -p ../epu-test-dir -``` - -### 3. Verify Prerequisites -```bash -# Check test recordings exist (workspace structure) -ls -lh ../../testdata/recordings/ - -# Verify k3s services accessible -curl -s http://localhost:30080/status || echo "API not ready yet" - -# Check database connectivity -psql -h localhost -p 30432 -U username -d smartem_db -c "SELECT version();" -``` - -## Test Execution Scenarios - -**CRITICAL**: Agent command is `python -m smartem_agent watch` (NOT `python -m smartem_agent.main`!) - -### Test Type 1: Post-Acquisition Data Intake -**Scenario**: Agent starts watching after playback completes (easiest, should work) - -```bash -# 1. Start backend services (ensure env vars are already exported from prep step) -python -m uvicorn smartem_backend.api_server:app --host 127.0.0.1 --port 8000 --log-level debug 2>&1 | tee "$TEST_DIR/logs/api.log" & -python -m smartem_backend.consumer -vv 2>&1 | tee "$TEST_DIR/logs/consumer.log" & - -# Wait a moment for services to start -sleep 3 - -# 2. Run playback to completion (no agent watching yet) -# NOTE: Recording must be a .tar.gz file (e.g., bi37708-42_fsrecord.tar.gz) -python ./tools/fsrecorder/fsrecorder.py replay \ - ../../testdata/recordings/bi37708-42_fsrecord.tar.gz \ - ../epu-test-dir 2>&1 | tee "$TEST_DIR/logs/playback.log" - -# 3. Wait for playback to finish, then start agent -# IMPORTANT: Use "python -m smartem_agent watch" NOT "python -m smartem_agent.main" -python -m smartem_agent watch \ - --api-url http://localhost:8000 \ - -vv \ - ../epu-test-dir 2>&1 | tee "$TEST_DIR/logs/agent.log" -``` - -### Test Type 2: Pre-Acquisition Agent Setup -**Scenario**: Agent starts watching before any data exists - -```bash -# 1. Start backend services -python -m uvicorn smartem_backend.api_server:app --host 127.0.0.1 --port 8000 --log-level debug 2>&1 | tee "$TEST_DIR/logs/api.log" & -python -m smartem_backend.consumer -vv 2>&1 | tee "$TEST_DIR/logs/consumer.log" & - -# Wait a moment for services to start -sleep 3 - -# 2. Start agent watching empty directory -# IMPORTANT: Use "python -m smartem_agent watch" NOT "python -m smartem_agent.main" -python -m smartem_agent watch \ - --api-url http://localhost:8000 \ - -vv \ - ../epu-test-dir 2>&1 | tee "$TEST_DIR/logs/agent.log" & - -# 3. Begin playback after agent is watching -# NOTE: Recording must be a .tar.gz file (e.g., bi37708-42_fsrecord.tar.gz) -python ./tools/fsrecorder/fsrecorder.py replay \ - ../../testdata/recordings/bi37708-42_fsrecord.tar.gz \ - ../epu-test-dir 2>&1 | tee "$TEST_DIR/logs/playback.log" -``` - -### Test Type 3: Mid-Acquisition Agent Start -**Scenario**: Agent starts after playback begins but before completion (tests handling existing + new data) - -**Timing considerations**: -- Playback script initially extracts data from recording (no files written yet) -- After extraction, playback begins writing files to output directory -- Agent should start after some files exist but before playback completes -- Use different delays to test at various stages of data availability - -```bash -# 1. Start backend services -python -m uvicorn smartem_backend.api_server:app --host 127.0.0.1 --port 8000 --log-level debug 2>&1 | tee "$TEST_DIR/logs/api.log" & -python -m smartem_backend.consumer -vv 2>&1 | tee "$TEST_DIR/logs/consumer.log" & - -# Wait a moment for services to start -sleep 3 - -# 2. Start playback first -# NOTE: Recording must be a .tar.gz file (e.g., bi37708-42_fsrecord.tar.gz) -python ./tools/fsrecorder/fsrecorder.py replay \ - ../../testdata/recordings/bi37708-42_fsrecord.tar.gz \ - ../epu-test-dir 2>&1 | tee "$TEST_DIR/logs/playback.log" & - -# 3. Monitor file creation to choose optimal start time -watch -n 1 "find ../epu-test-dir -type f | wc -l" - -# 4. When desired number of files exist, start agent -# Experiment with different timing: -# - sleep 5 # Very early (minimal files) -# - sleep 15 # Early-mid stage -# - sleep 30 # Mid-stage -# - sleep 60 # Late stage -sleep 15 # Adjust based on monitoring above - -# IMPORTANT: Use "python -m smartem_agent watch" NOT "python -m smartem_agent.main" -python -m smartem_agent watch \ - --api-url http://localhost:8000 \ - -vv \ - ../epu-test-dir 2>&1 | tee "$TEST_DIR/logs/agent.log" -``` - -## Common Mistakes to Avoid - -**ALWAYS check the agent log file first if things aren't working!** - -1. **Wrong agent command**: Use `python -m smartem_agent watch`, NOT: - - ❌ `python -m smartem_agent.main` - - ❌ `python -m smartem_agent` - - ✅ `python -m smartem_agent watch` - -2. **Wrong recording path**: Must use .tar.gz file, NOT directory: - - ❌ `../../testdata/recordings/pre-acquisition` - - ✅ `../../testdata/recordings/bi37708-42_fsrecord.tar.gz` - -3. **FastAPI dev server**: Use uvicorn for e2e tests, NOT `python -m fastapi dev`: - - ❌ `python -m fastapi dev src/smartem_backend/api_server.py` - - ✅ `python -m uvicorn smartem_backend.api_server:app` - -4. **Forgot to activate venv**: All commands must run with venv activated: - - ✅ `source .venv/bin/activate` at the start - -5. **Agent log is tiny (< 1KB)**: Agent failed to start - check the error message in agent.log - -## Post-Test Data Collection - -After test completion, capture all relevant data for analysis: - -```bash -# 1. Verify filesystem vs database counts -echo "=== Filesystem Counts ===" -echo "EPU Sessions (Supervisor_* dirs): $(ls -1d ../epu-test-dir/Supervisor_* 2>/dev/null | wc -l)" -echo "GridSquare directories: $(find ../epu-test-dir -type d -name "GridSquare_*" | wc -l)" -echo "FoilHole directories: $(find ../epu-test-dir -type d -name "FoilHole_*" | wc -l)" -echo "MRC files (micrographs): $(find ../epu-test-dir -type f -name "*.mrc" | wc -l)" -echo "XML metadata files: $(find ../epu-test-dir -type f -name "*.xml" | wc -l)" -echo "" -echo "=== Database Counts ===" -kubectl exec -n smartem-decisions deployment/postgres -- psql -U username -d smartem_db -c \ - "SELECT 'acquisition' as table_name, COUNT(*) as count FROM acquisition - UNION ALL SELECT 'grid', COUNT(*) FROM grid - UNION ALL SELECT 'gridsquare', COUNT(*) FROM gridsquare - UNION ALL SELECT 'foilhole', COUNT(*) FROM foilhole - UNION ALL SELECT 'micrograph', COUNT(*) FROM micrograph - ORDER BY table_name;" -echo "" -echo "=== Agent Processing Issues ===" -echo "No grid found messages: $(grep -c 'No grid found' "$TEST_DIR/logs/agent.log" 2>/dev/null || echo 0)" - -# 2. Database dump (final state after all processing) -kubectl exec -n smartem-decisions deployment/postgres -- pg_dump -U username -d smartem_db > "$TEST_DIR/db-dump.sql" - -# 3. Create test parameters record -cat > "$TEST_DIR/test-params.json" << EOF -{ - "test_type": "post-acquisition", - "timestamp": "$(date -Iseconds)", - "recording": "bi37708-42_fsrecord.tar.gz", - "playback_mode": "fast", - "agent_verbosity": "debug", - "consumer_verbosity": "debug", - "api_log_level": "debug", - "notes": "Testing data intake after microscope session completes" -} -EOF - -# 4. Verify all 4 service logs were captured -ls -lh "$TEST_DIR/logs/" -# Expected files: -# - playback.log (microscope recording playback) -# - agent.log (SmartEM agent) -# - api.log (backend HTTP API) -# - consumer.log (backend worker/consumer) - -# 5. Optional: Capture k8s infrastructure logs if debugging issues -kubectl logs deployment/postgres -n smartem-decisions > "$TEST_DIR/logs/postgres.log" -kubectl logs deployment/rabbitmq -n smartem-decisions > "$TEST_DIR/logs/rabbitmq.log" -``` - -**Critical data for analysis**: -1. **Filesystem vs Database counts**: Verify data ingestion completeness -2. **playback.log**: File creation timeline, extraction timing -3. **agent.log**: File detection, API communication, SSE events, "No grid found" errors -4. **api.log**: HTTP requests, database writes, SSE connections -5. **consumer.log**: RabbitMQ message processing, prediction updates -6. **db-dump.sql**: Final database state (acquisitions, grids, predictions) - -## Monitoring and Verification - -### During Test Execution - -**Check file creation**: -```bash -watch -n 2 "find ../epu-test-dir -type f | wc -l" -``` - -**Monitor API status**: -```bash -# Check acquisitions being created -curl -s http://localhost:8000/acquisitions | jq - -# Check overall status -curl -s http://localhost:8000/status | jq -``` - -**Watch logs in real-time**: -```bash -tail -f "$TEST_DIR/logs/agent.log" -tail -f "$TEST_DIR/logs/consumer.log" -tail -f "$TEST_DIR/logs/playback.log" -``` - -### Success Criteria - -A successful test should demonstrate: -- ✅ All backend services start without errors -- ✅ RabbitMQ queue is empty at test start -- ✅ Database is freshly created with no leftover data -- ✅ Playback creates expected files in output directory -- ✅ Agent detects file changes and sends data to API -- ✅ API receives and stores data in database -- ✅ Worker processes messages from queue -- ✅ No errors in any service logs -- ✅ Database contains expected entities after test -- ✅ **Filesystem counts match database counts**: EPU sessions → grids, GridSquares → gridsquare rows, MRC files → micrograph rows -- ✅ **No or minimal "No grid found" messages** in agent.log (indicates proper grid association) - -## Cleanup and Teardown - -### Stop Running Services -```bash -# Stop all test-related processes -pkill -f smartem_backend.api -pkill -f smartem_backend.consumer -pkill -f smartem_agent -pkill -f fsrecorder - -# Verify processes stopped -pgrep -f "smartem_backend|smartem_agent|fsrecorder" -``` - -### Optional: Full Teardown -```bash -# Clean playback directory -rm -rf ../epu-test-dir - -# Stop k3s cluster (if needed) -./tools/k8s/dev-k8s.sh down -``` - -## Troubleshooting - -### Database Connection Issues -```bash -# Verify k3s postgres is accessible -kubectl get pods -n smartem-decisions -kubectl logs deployment/postgres -n smartem-decisions - -# Test connection -psql -h localhost -p 30432 -U username -d smartem_db -c "\l" -``` - -### RabbitMQ Connection Issues -```bash -# Check RabbitMQ pod status -kubectl get pods -n smartem-decisions -kubectl logs deployment/rabbitmq -n smartem-decisions - -# Verify connection from host -curl -u guest:guest http://localhost:30673 # pragma: allowlist secret -``` - -### Agent Not Detecting Files -- Ensure `--api-url http://localhost:8000` points to correct API endpoint -- Check agent logs for connection errors -- Verify playback is writing to the directory agent is watching -- Confirm file permissions allow agent to read files - -### Consumer Not Processing Messages -- Verify `RABBITMQ_URL=amqp://guest:guest@localhost:30672/` is correct -- Check queue has messages waiting -- Review consumer logs for connection errors -- Ensure database is accessible to worker - -## Known Issues and Notes - -**RabbitMQ Queue Persistence**: -- Queue `smartem_backend` is durable and persists messages across connections -- **If multiple workers are started, they may compete for messages** - verify this is intended behaviour -- Restarting RabbitMQ service is currently the cleanest way to ensure empty queue between tests -- Consider implementing programmatic queue purging if restart is too slow - -## Notes for Future Refinement - -This document will evolve as we: -- Complete missing command details -- Identify optimal test parameters -- Discover edge cases during testing -- Convert manual steps into automation scripts -- Potentially create custom slash commands for common workflows diff --git a/webui/src/docs/how-to/sync-github-labels.mdx b/webui/src/docs/how-to/sync-github-labels.mdx deleted file mode 100644 index 7ab4ed0..0000000 --- a/webui/src/docs/how-to/sync-github-labels.mdx +++ /dev/null @@ -1,159 +0,0 @@ -# How to Sync GitHub Labels - -This guide explains how to manage GitHub labels across SmartEM repositories using the label sync tool. - -## Overview - -The SmartEM ecosystem uses a standardised set of GitHub labels across four repositories: - -- `DiamondLightSource/smartem-decisions` -- `DiamondLightSource/smartem-frontend` -- `DiamondLightSource/smartem-devtools` -- `DiamondLightSource/fandanGO-cryoem-dls` - -Labels are defined in `core/github-tags-config.ts` and synced using `tools/github/sync-labels.ts`. - -## Prerequisites - -- Node.js 18+ -- GitHub CLI (`gh`) installed and authenticated -- For sync operations: write access to target repositories - -## Label Categories - -### Types of Work - -Labels that categorise the nature of work being done: - -| Label | Colour | Purpose | -|-------|--------|---------| -| documentation | teal | Improvements or additions to project documentation | -| testing | green | Writing, updating, or fixing automated tests | -| bugfixing | red | Fixing defects or unexpected behavior | -| development | purple | New features or functionality implementation | -| refactoring | orange | Code restructuring without changing behavior | -| research | cyan | Investigation, spikes, or proof-of-concept work | -| devops | slate | CI/CD, deployment, infrastructure, or tooling | -| security | maroon | Security fixes, audits, or vulnerability remediation | -| admin | brown | Project maintenance, dependency updates | -| enhancement | dark cyan | Minor improvements to existing functionality | - -### System Components - -Labels that identify which part of the system is affected: - -| Label | Colour Family | Purpose | -|-------|---------------|---------| -| smartem-backend | Blue (dark) | Core backend services | -| smartem-backend:db | Blue (mid) | Database and data layer | -| smartem-backend:api | Blue (light) | REST API endpoints | -| smartem-agent | Gold | EPU workstation agent | -| smartem-frontend | Green | User-facing web UI | -| smartem-aria-connector | Purple | ARIA/FandanGO integration | -| smartem-devtools | Pink (dark) | Developer tooling | -| smartem-devtools:webui | Pink (mid) | Developer dashboard | -| smartem-devtools:claude | Pink (light) | Claude Code configuration | -| smartem-devtools:e2e-test | Pink (lightest) | E2E testing infrastructure | - -## Usage - -### Check Label Conformity - -To check if all repositories have the correct labels without making changes: - -```bash -npm run labels:check -``` - -Or directly: - -```bash -npx tsx tools/github/sync-labels.ts --check -``` - -This will: -1. Fetch existing labels from each repository -2. Compare against the defined labels in `core/github-tags-config.ts` -3. Report any discrepancies (missing, extra, or outdated labels) -4. Exit with code 1 if any repository is non-conforming - -### Sync Labels - -To synchronise labels across all repositories: - -```bash -npm run labels:sync -``` - -Or directly: - -```bash -npx tsx tools/github/sync-labels.ts --sync -``` - -This will: -1. Delete labels not in the definition (extra labels) -2. Create labels that are missing -3. Update labels where description or colour has changed - -### Target Specific Repositories - -To sync only specific repositories: - -```bash -npx tsx tools/github/sync-labels.ts --sync --repo smartem-decisions -npx tsx tools/github/sync-labels.ts --sync --repo smartem-frontend --repo smartem-devtools -``` - -### Verbose Output - -For detailed output including conforming labels: - -```bash -npx tsx tools/github/sync-labels.ts --check --verbose -``` - -## CI/CD Integration - -The `gitflow.yml` workflow automates label management: - -- **On push to main**: Runs `--check` mode when `core/github-tags-config.ts` or `tools/github/**` changes -- **Manual dispatch**: Can trigger `--sync` mode via GitHub Actions UI - -### Running Sync via CI/CD - -1. Go to Actions tab in smartem-devtools repository -2. Select "Gitflow" workflow -3. Click "Run workflow" -4. Select `sync` mode -5. Click "Run workflow" - -## Modifying Labels - -To add, remove, or modify labels: - -1. Edit `core/github-tags-config.ts` -2. Run `npm run labels:check` to verify changes -3. Run `npm run labels:sync` to apply changes -4. Commit and push changes - -The CI/CD workflow will verify conformity on push. - -## Troubleshooting - -### Authentication Errors - -Ensure `gh` CLI is authenticated: - -```bash -gh auth status -gh auth login # if not authenticated -``` - -### Permission Denied - -For sync operations, you need write access to all target repositories. The CI/CD workflow uses a PAT stored as `LABEL_SYNC_TOKEN` secret. - -### Label Already Exists - -If a label creation fails because it already exists, the script will continue. Run with `--verbose` to see details. diff --git a/webui/src/docs/how-to/troubleshoot-cli.mdx b/webui/src/docs/how-to/troubleshoot-cli.mdx deleted file mode 100644 index 1a2f568..0000000 --- a/webui/src/docs/how-to/troubleshoot-cli.mdx +++ /dev/null @@ -1,558 +0,0 @@ -# SmartEM CLI Troubleshooting Guide - -This guide provides solutions for common issues encountered when using the SmartEM Agent command-line interface. For comprehensive parameter documentation, see the [CLI Reference](../reference/cli-reference.md). - -## Quick Diagnostics - -### Check CLI Installation -```bash -# Verify the CLI is accessible -python -m smartem_agent --help - -# Check version and dependencies -python -c "import smartem_agent; print('SmartEM Agent available')" -``` - -### Test Basic Functionality -```bash -# Test with a known good directory -python -m smartem_agent validate /path/to/test/data - -# Test API connectivity -python -m smartem_agent watch /tmp --dry-run --verbose -``` - -## Common Issues and Solutions - -### 1. Command Not Found Errors - -#### Error: `No module named 'smartem_agent'` - -**Cause:** SmartEM Agent package is not installed or not in Python path. - -**Solutions:** -```bash -# Install in development mode -pip install -e . - -# Install with all dependencies -pip install -e .[all] - -# Verify installation -pip list | grep smartem -``` - -**Alternative:** Use the full path to the module: -```bash -PYTHONPATH=/path/to/smartem-decisions/src python -m smartem_agent --help -``` - -#### Error: `python: can't open file 'smartem_agent'` - -**Cause:** Trying to run as a script instead of a module. - -**Solution:** Use the module syntax: -```bash -# Correct -python -m smartem_agent watch /data - -# Incorrect -python smartem_agent watch /data -``` - -### 2. Directory and File Access Issues - -#### Error: `Directory /path/to/data does not exist` - -**Diagnosis:** -```bash -# Check if directory exists -ls -la /path/to/data - -# Check parent directory -ls -la /path/to/ - -# Verify current working directory -pwd -``` - -**Solutions:** -- Use absolute paths: `/full/path/to/directory` -- Verify directory spelling and case sensitivity -- Check directory permissions: `ls -ld /path/to/data` - -#### Error: `Permission denied` - -**Cause:** Insufficient permissions to read directories or write log files. - -**Solutions:** -```bash -# Check directory permissions -ls -la /path/to/data - -# Check write permissions for log file location -touch /path/to/logfile.log && rm /path/to/logfile.log - -# Run with appropriate permissions -sudo python -m smartem_agent watch /data # Use sparingly - -# Better: Fix directory permissions -chmod -R 755 /path/to/data -chown -R $USER:$GROUP /path/to/data -``` - -#### Error: Files not being detected during watch - -**Diagnosis:** -```bash -# Test with verbose output -python -m smartem_agent watch /data --dry-run -vv - -# Check file patterns match -find /data -name "EpuSession.dm" -o -name "*.xml" - -# Monitor filesystem events manually -inotifywait -m -r /data -``` - -**Solutions:** -- Verify file patterns match EPU naming conventions -- Check for symbolic links that might not be followed -- Ensure files are completely written before processing -- Consider filesystem-specific issues (NFS, network drives) - -### 3. API Connection Problems - -#### Error: `API at http://127.0.0.1:8000 is not reachable` - -**Diagnosis:** -```bash -# Test basic connectivity -curl -v http://127.0.0.1:8000/status - -# Check if service is running -netstat -tlnp | grep 8000 - -# Test from Python -python -c "import requests; print(requests.get('http://127.0.0.1:8000/status').json())" -``` - -**Solutions:** - -1. **Start the backend API:** - ```bash - cd /path/to/smartem-decisions - python -m smartem_backend.api_server - ``` - -2. **Use correct API URL:** - ```bash - # Local development - python -m smartem_agent watch /data --api-url http://localhost:8000 - - # Remote server - python -m smartem_agent watch /data --api-url https://smartem.example.com/api - ``` - -3. **Network connectivity issues:** - - Check firewall settings - - Verify proxy configuration - - Test with different network interfaces - - Use `--dry-run` for offline testing - -#### Error: `SSE connection failed` or `Failed to send heartbeat` - -**Diagnosis:** -```bash -# Test SSE endpoint manually -curl -H "Accept: text/event-stream" \ - http://127.0.0.1:8000/agent/test-agent/session/test-session/instructions/stream - -# Check heartbeat endpoint -curl -X POST \ - -H "Content-Type: application/json" \ - http://127.0.0.1:8000/agent/test-agent/session/test-session/heartbeat -``` - -**Solutions:** - -1. **Verify agent and session IDs:** - ```bash - # Ensure IDs exist in backend - python -m smartem_agent watch /data \ - --agent-id valid-agent-id \ - --session-id valid-session-id - ``` - -2. **Adjust timeout settings:** - ```bash - # Increase timeouts for unstable connections - python -m smartem_agent watch /data \ - --agent-id agent-01 \ - --session-id session-01 \ - --sse-timeout 120 \ - --heartbeat-interval 30 - ``` - -3. **Check backend logs:** - - Monitor backend API logs for connection errors - - Verify database connectivity - - Check for authentication issues - -### 4. Parsing and Validation Errors - -#### Error: `Grid data dir is structurally invalid` - -**Diagnosis:** -```bash -# Get detailed validation errors -python -m smartem_agent validate /path/to/grid -vv - -# Check directory structure -find /path/to/grid -type f -name "*.dm" -o -name "*.xml" | head -20 -``` - -**Common Issues:** -- Missing `EpuSession.dm` file -- Incorrect directory naming conventions -- Incomplete or corrupted files -- Wrong directory structure (not EPU format) - -**Solutions:** -- Verify the directory contains valid EPU data -- Check for required files: `EpuSession.dm`, `Atlas/Atlas.dm` -- Ensure directory structure matches EPU conventions -- Use `parse dir` command to identify specific parsing issues - -#### Error: `Could not extract instrument info` or parsing failures - -**Diagnosis:** -```bash -# Test individual file parsing -python -m smartem_agent parse session /path/to/EpuSession.dm -vv - -# Check file integrity -file /path/to/EpuSession.dm -hexdump -C /path/to/EpuSession.dm | head -5 -``` - -**Solutions:** -- Verify files are not corrupted or partially written -- Check file permissions and accessibility -- Ensure files are in expected format (not binary corrupted) -- Look for encoding issues or special characters in paths - -### 5. Performance and Resource Issues - -#### Issue: High memory usage or slow processing - -**Diagnosis:** -```bash -# Monitor resource usage -top -p $(pgrep -f "smartem_agent") -htop - -# Check directory size -du -sh /path/to/data -find /path/to/data -type f | wc -l -``` - -**Solutions:** - -1. **Optimise logging settings:** - ```bash - # Reduce logging frequency - python -m smartem_agent watch /data --log-interval 30.0 - - # Reduce verbosity - python -m smartem_agent watch /data # No -v flags - ``` - -2. **Process smaller datasets:** - ```bash - # Process single grids instead of entire sessions - python -m smartem_agent watch /data/Grid_001 - ``` - -3. **System resource limits:** - ```bash - # Increase file descriptor limits - ulimit -n 4096 - - # Monitor disk space - df -h /path/to/data - ``` - -#### Issue: `OSError: [Errno 24] Too many open files` - -**Solutions:** -```bash -# Check current limits -ulimit -a - -# Increase file descriptor limit temporarily -ulimit -n 4096 - -# Increase permanently (add to ~/.bashrc) -echo "ulimit -n 4096" >> ~/.bashrc -``` - -### 6. Logging and Output Issues - -#### Issue: No log output or missing log files - -**Diagnosis:** -```bash -# Check log file permissions -ls -la fs_changes.log - -# Verify log directory exists -ls -la $(dirname /path/to/custom.log) - -# Test with different log location -python -m smartem_agent watch /data --log-file /tmp/test.log -``` - -**Solutions:** -- Ensure log directory exists and is writable -- Use absolute paths for log files -- Check disk space availability -- Verify SELinux/AppArmor policies if applicable - -#### Issue: Verbose output not showing - -**Solutions:** -```bash -# Ensure correct verbose syntax -python -m smartem_agent watch /data -v # INFO level -python -m smartem_agent watch /data -vv # DEBUG level - -# Check if output is being redirected -python -m smartem_agent watch /data -v 2>&1 | tee output.log -``` - -### 7. Signal Handling and Process Management - -#### Issue: Process doesn't stop gracefully with Ctrl+C - -**Solutions:** -```bash -# Send SIGTERM for graceful shutdown -pkill -TERM -f "smartem_agent" - -# Force kill if necessary -pkill -9 -f "smartem_agent" - -# Use timeout for automatic termination -timeout 3600 python -m smartem_agent watch /data # Stop after 1 hour -``` - -#### Issue: Background process monitoring - -**Solutions:** -```bash -# Run in background with nohup -nohup python -m smartem_agent watch /data > smartem.log 2>&1 & - -# Use screen or tmux for persistent sessions -screen -S smartem -python -m smartem_agent watch /data - -# Monitor process status -ps aux | grep smartem_agent -``` - -## Advanced Troubleshooting - -### Debug Mode Activation - -Enable maximum debugging output: -```bash -python -m smartem_agent watch /data \ - --dry-run \ - --verbose --verbose \ - --log-interval 1.0 \ - --heartbeat-interval 10 \ - --sse-timeout 10 -``` - -### Network Debugging - -**Test connectivity chain:** -```bash -# 1. Basic network connectivity -ping backend-host - -# 2. Port connectivity -telnet backend-host 8000 - -# 3. HTTP connectivity -curl -v http://backend-host:8000/status - -# 4. SSE connectivity -curl -v -H "Accept: text/event-stream" \ - http://backend-host:8000/agent/test/session/test/instructions/stream -``` - -### File System Monitoring Debug - -**Manual file monitoring:** -```bash -# Install inotify-tools (Linux) -sudo apt-get install inotify-tools - -# Monitor directory changes -inotifywait -m -r --format '%w%f %e %T' --timefmt '%Y-%m-%d %H:%M:%S' /data - -# Compare with agent detection -python -m smartem_agent watch /data --dry-run -vv -``` - -### Database Connectivity Issues - -**Test database connection (if using persistent storage):** -```bash -# Check database connectivity -psql -h database-host -U username -d smartem_db -c "SELECT 1;" - -# Verify tables exist -psql -h database-host -U username -d smartem_db -c "\\dt" - -# Check for connection pooling issues -netstat -an | grep :5432 -``` - -## Environment-Specific Issues - -### Windows Specific - -**Path separator issues:** -```bash -# Use forward slashes or raw strings -python -m smartem_agent watch "C:/data/microscopy" -python -m smartem_agent watch C:\\data\\microscopy -``` - -**Service account permissions:** -- Ensure service account has appropriate file system access -- Check Windows Defender exclusions for monitoring directories -- Verify no Group Policy restrictions on file access - -### Linux/Unix Specific - -**Permission issues:** -```bash -# Check SELinux status -sestatus - -# Check AppArmor status -sudo apparmor_status - -# Verify no systemd restrictions -systemctl status user@$(id -u).service -``` - -**NFS mounted directories:** -```bash -# Check mount options -mount | grep nfs - -# Test with local directory first -python -m smartem_agent watch /tmp/test_data --dry-run -``` - -### Container/Docker Environments - -**Volume mounting issues:** -```bash -# Verify volume mounts -docker exec container-id ls -la /data - -# Check container permissions -docker exec container-id id -docker exec container-id ls -la /data -``` - -**Network connectivity in containers:** -```bash -# Test from within container -docker exec container-id curl http://backend:8000/status - -# Check container networking -docker network ls -docker inspect network-name -``` - -## Getting Additional Help - -### Collecting Diagnostic Information - -Create a diagnostic script: -```bash -#!/bin/bash -echo "=== System Information ===" -uname -a -python --version - -echo "=== SmartEM Agent Status ===" -python -c "import smartem_agent; print('Available')" 2>&1 - -echo "=== Directory Information ===" -ls -la /path/to/data - -echo "=== Network Connectivity ===" -curl -s http://127.0.0.1:8000/status 2>&1 || echo "API not reachable" - -echo "=== Process Information ===" -ps aux | grep smartem - -echo "=== Resource Usage ===" -free -h -df -h -``` - -### Log Analysis - -**Extract relevant log entries:** -```bash -# Filter for errors -grep -i error fs_changes.log - -# Filter by time range -awk '/2024-01-15 14:00:00/,/2024-01-15 15:00:00/' fs_changes.log - -# Analyse patterns -grep "Heartbeat sent" fs_changes.log | tail -20 -``` - -### Reporting Issues - -When reporting issues, include: - -1. **Command used:** Full command line with parameters -2. **Error message:** Complete error output -3. **Environment:** OS, Python version, installation method -4. **Directory structure:** Sample of the directory being processed -5. **Logs:** Relevant log entries with timestamps -6. **Network information:** API URLs, connectivity status -7. **System resources:** Available memory, disk space - -**Example issue report:** -``` -Command: python -m smartem_agent watch /data/Grid_001 --agent-id microscope-01 --session-id session-123 -v - -Error: SSE connection failed: Connection refused - -Environment: -- OS: Ubuntu 20.04 LTS -- Python: 3.12.1 -- Installation: pip install -e .[all] - -Directory: /data/Grid_001 (contains EpuSession.dm, 15 GridSquare directories) - -API Status: curl http://127.0.0.1:8000/status returns {"status": "ok"} - -Logs: (attach relevant log entries) -``` - -This comprehensive troubleshooting guide should resolve most common CLI issues. For backend-specific problems, consult the SmartEM Backend documentation. diff --git a/webui/src/docs/how-to/use-api-documentation.mdx b/webui/src/docs/how-to/use-api-documentation.mdx deleted file mode 100644 index e31392f..0000000 --- a/webui/src/docs/how-to/use-api-documentation.mdx +++ /dev/null @@ -1,205 +0,0 @@ -# Using the API Documentation - -This guide explains how to use the interactive API documentation for SmartEM Decisions services. - -## Overview - -SmartEM Decisions provides interactive API documentation built with Swagger UI, giving you: - -- **Live API exploration** - Try endpoints directly in your browser -- **Complete specifications** - Full OpenAPI 3.0 documentation -- **Example requests/responses** - See exactly what data to send and expect -- **Authentication helpers** - Built-in tools for API authentication - -## Accessing the Documentation - -### Online Documentation - -Visit the hosted API documentation at: -- **Athena Decision Service**: [Athena API Docs](../api/athena/index.html) -- **SmartEM Core API**: [SmartEM API Docs](../api/smartem/index.html) - -### Local Documentation - -To run the documentation locally with a live mock server: - -```bash -# Install mock dependencies -pip install -e ".[mock]" - -# Start the Athena mock server -python -c " -from athena_api.mock import AthenaAPIServer -server = AthenaAPIServer() -server.run() -" - -# Visit http://localhost:8000/docs -``` - -## Using the Interactive Features - -### Try It Out - -1. **Expand an endpoint** by clicking on it -2. **Click "Try it out"** to enable the interactive form -3. **Fill in parameters** using the provided form fields -4. **Click "Execute"** to make a real API call -5. **View the response** including status code, headers, and body - -### Authentication - -For endpoints requiring authentication: - -1. **Click the "Authorize" button** at the top of the page -2. **Enter your credentials** (API key, bearer token, etc.) -3. **Click "Authorize"** to save credentials for all requests - -### Request Examples - -Each endpoint shows: -- **Parameter descriptions** - What each field does -- **Example values** - Sample data to help you understand the format -- **Schema definitions** - Complete data structure specifications -- **Response examples** - What to expect back from the API - -## API Specifications - -### Understanding API Sources - -**Athena Decision Service API** - External service integration: -- **Source of Truth**: External Athena service OpenAPI specification (`docs/athena-decision-service-api-spec.json`) -- **Our Implementation**: Python client and mock server generated from spec -- **Documentation**: Based on the authoritative external specification - -**SmartEM Core API** - Our implementation: -- **Source of Truth**: Our FastAPI/Django implementation -- **Our Implementation**: Backend service with business logic -- **Documentation**: Generated from our actual API endpoints - -### Download Specifications - -You can download the raw OpenAPI specifications: - -- [Athena API Spec](../api/athena/swagger.json) - Official external specification -- [Athena Source Spec](../athena-decision-service-api-spec.json) - Original specification file -- [SmartEM API Spec](../api/smartem/swagger.json) - Generated from our implementation - -### Using Specifications - -Import these into your favorite API tools: -- **Postman** - Import → Link → Paste URL -- **Insomnia** - Create → Import/Export → From URL -- **VS Code REST Client** - Use with OpenAPI extensions -- **Code generators** - Generate client libraries in various languages - -## Common Workflows - -### Testing Decision Service Integration - -1. **Start with session management**: - ```bash - POST /api/v1/Session - # Register a new acquisition session - ``` - -2. **Register areas**: - ```bash - POST /api/v1/Area - # Add grid squares and foil holes - ``` - -3. **Record decisions**: - ```bash - POST /api/v1/Decision - # Log automated decisions - ``` - -4. **Store algorithm results**: - ```bash - POST /api/v1/AlgorithmResult - # Save processing outcomes - ``` - -### Monitoring Acquisition Status - -1. **Check application state**: - ```bash - GET /api/v1/CurrentApplicationState - # Get current system status - ``` - -2. **Query area states**: - ```bash - GET /api/v1/Session/{sessionId}/AreaStates - # Monitor area processing status - ``` - -3. **Retrieve decisions**: - ```bash - GET /api/v1/Session/{sessionId}/Decisions - # Review automated decisions - ``` - -## Development Tips - -### Mock Server for Development - -The Athena API includes a full mock server for development: - -```python -from athena_api.mock import AthenaAPIServer - -# Create and configure mock server -server = AthenaAPIServer() - -# Customize host/port -server.run(host="0.0.0.0", port=8080) -``` - -### Client Library Usage - -Use the Python client library for programmatic access: - -```python -from athena_api import AthenaClient -from athena_api.model.request import Session -from uuid import uuid4 -from datetime import datetime - -# Connect to API -client = AthenaClient("http://localhost:8000") - -# Create a session -session = Session( - sessionId=uuid4(), - sessionName="Test Session", - timestamp=datetime.now(), - gridType="HoleyCarbon" -) - -# Register the session -response = client.register_session(session) -print(f"Session created: {response.sessionId}") -``` - -### Error Handling - -The API follows standard HTTP status codes: - -- **200-299**: Success responses -- **400-499**: Client errors (bad requests, missing data) -- **500-599**: Server errors (system failures) - -Check the API documentation for specific error responses and how to handle them. - -## Support - -For API-related questions: - -1. **Check the interactive documentation** for endpoint details -2. **Review the example code** in this documentation -3. **Examine the client library** source code for usage patterns -4. **Open an issue** on the [GitHub repository](https://github.com/DiamondLightSource/smartem-decisions/issues) - -The API documentation is automatically updated with each release, ensuring you always have access to the latest features and changes. diff --git a/webui/src/docs/how-to/use-core-http-api-client.mdx b/webui/src/docs/how-to/use-core-http-api-client.mdx deleted file mode 100644 index e685414..0000000 --- a/webui/src/docs/how-to/use-core-http-api-client.mdx +++ /dev/null @@ -1,256 +0,0 @@ -# How to Use the SmartEM API Client - -This guide explains how to use the unified SmartEM API client to communicate with the SmartEM Core API. - -## Overview - -The SmartEM API Client provides a unified interface to interact with the SmartEM Core API. It supports both -synchronous and asynchronous operations, making it flexible for different usage scenarios. The client also includes -data conversion utilities to convert between EPU data models and API request/response models. - -## Installation - -The SmartEM API Client is included with the SmartEM Decisions package, so no additional installation is required. - -## Basic Usage - -### Importing the Client - -```python -from smartem_backend.api_client import SmartEMAPIClient -``` - -### Creating a Client Instance - -```python -# Create a client with default timeout (10 seconds) -client = SmartEMAPIClient("http://localhost:8000") - -# Create a client with custom timeout -client = SmartEMAPIClient("http://localhost:8000", timeout=30.0) - -# Create a client with custom logger -import logging -logger = logging.getLogger("my_logger") -client = SmartEMAPIClient("http://localhost:8000", logger=logger) -``` - -### Using the Client with Context Managers - -The client supports context managers to ensure proper resource cleanup: - -```python -# Synchronous context manager -with SmartEMAPIClient("http://localhost:8000") as client: - # Use the client... - status = client.get_status() - print(status) - -# Asynchronous context manager -async with SmartEMAPIClient("http://localhost:8000") as client: - # Use the client asynchronously... - status = await client.aget_status() - print(status) -``` - -## Synchronous vs Asynchronous Methods - -The client provides both synchronous and asynchronous methods for all operations. The asynchronous methods are -prefixed with `a` (e.g., `aget_status` vs `get_status`). - -```python -# Synchronous method -acquisitions = client.get_acquisitions() - -# Asynchronous method -acquisitions = await client.aget_acquisitions() -``` - -## Working with Acquisitions - -### Creating an Acquisition - -```python -from smartem_agent.model.schemas import EpuSessionData -from datetime import datetime - -# Create an acquisition from EPU session data -session = EpuSessionData( - id="my-acquisition-id", - name="My Acquisition", - start_time=datetime.now(), - storage_path="/path/to/storage" -) -response = client.create_acquisition(session) -print(f"Created acquisition with ID: {response.id}") - -# Alternatively, create directly with an API request model -from smartem_backend.model.http_request import AcquisitionCreateRequest - -request = AcquisitionCreateRequest( - id="my-acquisition-id", - name="My Acquisition" -) -response = client.create_acquisition(request) -``` - -### Retrieving Acquisitions - -```python -# Get all acquisitions -acquisitions = client.get_acquisitions() -for acq in acquisitions: - print(f"Acquisition: {acq.id} - {acq.name}") - -# Get a specific acquisition -acquisition = client.get_acquisition("acquisition-id") -print(f"Acquisition details: {acquisition.name}, Status: {acquisition.status}") -``` - -### Updating an Acquisition - -```python -from smartem_backend.model.http_request import AcquisitionUpdateRequest - -update = AcquisitionUpdateRequest( - name="Updated Acquisition Name", - status="completed" -) -updated = client.update_acquisition("acquisition-id", update) -``` - -### Deleting an Acquisition - -```python -client.delete_acquisition("acquisition-id") -``` - -## Working with Hierarchical Data - -The client supports the full hierarchy of entities: - -1. Acquisition -2. Grid -3. Grid Square -4. Foil Hole -5. Micrograph - -Here's an example of creating entities at each level: - -```python -# Create an acquisition -acquisition = client.create_acquisition(EpuSessionData(id="acq-1", name="Test Acquisition")) - -# Create a grid for the acquisition -from smartem_agent.model.schemas import GridData - -grid = GridData(data_dir="/path/to/grid") -grid_response = client.create_acquisition_grid(acquisition.id, grid) - -# Create a grid square for the grid -from smartem_agent.model.schemas import GridSquareData - -gridsquare = GridSquareData(id="gs-1", data_dir="/path/to/gridsquare") -gridsquare_response = client.create_grid_gridsquare(grid_response.id, gridsquare) - -# Create a foil hole for the grid square -from smartem_agent.model.schemas import FoilHoleData - -foilhole = FoilHoleData(id="fh-1", gridsquare_id=gridsquare.id) -foilhole_response = client.create_gridsquare_foilhole(gridsquare_response.id, foilhole) - -# Create a micrograph for the foil hole -from smartem_agent.model.schemas import MicrographData, MicrographManifest - -manifest = MicrographManifest( - unique_id="mic-1", - acquisition_datetime=datetime.now(), - detector_name="K3", - energy_filter=True, - phase_plate=False, - binning_x=1, - binning_y=1 -) -micrograph = MicrographData( - id="mic-1", - gridsquare_id=gridsquare.id, - foilhole_id=foilhole.id, - location_id="loc-1", - high_res_path="/path/to/mic.mrc", - manifest_file="/path/to/manifest.xml", - manifest=manifest -) -micrograph_response = client.create_foilhole_micrograph(foilhole_response.id, micrograph) -``` - -## EntityStore Compatibility - -The client maintains compatibility with the existing `EntityStore` API for seamless integration: - -```python -# Create an entity through the EntityStore compatibility API -success = client.create("acquisition", "acq-id", session_data, parent=None) - -# Create a grid with parent relationship -success = client.create("grid", "grid-id", grid_data, parent=("acquisition", "acq-id")) -``` - -## Error Handling - -The client includes comprehensive error handling: - -```python -try: - response = client.get_acquisition("non-existent-id") -except httpx.HTTPStatusError as e: - print(f"HTTP error {e.response.status_code}: {e}") -except Exception as e: - print(f"Error: {e}") -``` - -## Logging - -The client includes detailed logging. You can configure the log level to control verbosity: - -```python -import logging -logging.basicConfig(level=logging.INFO) # For normal operation -logging.basicConfig(level=logging.DEBUG) # For more detailed logs including request/response data -``` - -## Advanced Usage - -### Working with Raw API Responses - -If you need to work with the raw API responses rather than parsed models: - -```python -raw_response = await client._request("get", "status") -print(raw_response) -``` - -### Managing the ID Mapping Cache - -The client maintains a cache of entity IDs to database IDs: - -```python -# Get a mapped database ID -db_id = client._get_db_id("acquisition", "entity-id") - -# Store a mapping manually (normally done automatically) -client._store_entity_id_mapping("acquisition", "entity-id", "db-id") -``` - -## Closing the Client - -Always close the client when you're done to free up resources: - -```python -# Explicitly close -client.close() - -# Or use context managers (recommended) -with SmartEMAPIClient("http://localhost:8000") as client: - # Client will be automatically closed when exiting the context - pass -``` diff --git a/webui/src/docs/how-to/use-mcp-interface.mdx b/webui/src/docs/how-to/use-mcp-interface.mdx deleted file mode 100644 index a0f11bb..0000000 --- a/webui/src/docs/how-to/use-mcp-interface.mdx +++ /dev/null @@ -1,507 +0,0 @@ -# Using SmartEM MCP Interface - -The SmartEM MCP (Model Context Protocol) interface provides natural language querying capabilities for microscopy session data. Built with FastMCP 2.0, it supports both filesystem-based parsing and API-based queries for comprehensive data access. - -## Overview - -The MCP interface consists of: -- **FastMCP Server**: Provides natural language query capabilities via MCP protocol using FastMCP framework -- **MCP Client**: Python client for programmatic access with simplified FastMCP patterns -- **CLI Interface**: Command-line tools for interactive usage - -## Installation - -Install with MCP dependencies (includes FastMCP 2.0 framework): - -```bash -pip install -e .[mcp] -``` - -This installs the required FastMCP 2.0 framework and SmartEM MCP components. - -## Configuration - -The MCP server can be configured using environment variables. A template configuration file is provided: - -```bash -# Copy the example configuration -cp .env.example.mcp .env.mcp - -# Edit .env.mcp with your settings -``` - -Available configuration options (see `.env.example.mcp` for details): -- `SMARTEM_MCP_DATA_PATH`: Path to EPU sessions directory (for filesystem adapter) -- `SMARTEM_MCP_API_URL`: SmartEM backend API URL (for API adapter) -- `SMARTEM_MCP_ADAPTER`: Data adapter selection (`filesystem` or `api`) -- `SMARTEM_MCP_LOG_LEVEL`: Logging level (`DEBUG`, `INFO`, `WARNING`, `ERROR`) - -## Quick Start - -### 1. Interactive Query Mode - -Start interactive mode for natural language questions: - -```bash -python -m smartem_mcp interactive -``` - -Example questions: -- "Show me a summary of session /path/to/epu/directory" -- "Find low quality items in /path/to/epu with threshold 0.3" -- "What are the recent acquisitions?" - -### 2. Command Line Usage - -Parse EPU directory: -```bash -python -m smartem_mcp client parse --path /path/to/epu/session -``` - -Find low quality items: -```bash -python -m smartem_mcp client quality --path /path/to/epu --threshold 0.3 -``` - -Query recent acquisitions (requires API): -```bash -python -m smartem_mcp client acquisitions --limit 10 -``` - -### 3. Programmatic Usage - -```python -from smartem_mcp.client import SmartEMMCPClient - -async with SmartEMMCPClient() as client: - # Parse EPU directory - result = await client.parse_epu_directory("/path/to/epu") - if result.get("success"): - print(f"Found {result['grid_count']} grids") - - # Find low quality items - quality_result = await client.find_low_quality_items( - "/path/to/epu", - threshold=0.3 - ) - - # Query API data - acquisitions = await client.query_recent_acquisitions(limit=5) -``` - -## Data Sources - -### Filesystem Queries - -Direct parsing of EPU XML files using existing `smartem_agent` tools: - -- **Use case**: Ad-hoc analysis, debugging, development -- **Capabilities**: Full EPU directory parsing, quality analysis -- **Requirements**: Direct filesystem access to EPU data - -### API Queries - -Query historical and in-flight sessions via SmartEM API: - -- **Use case**: Historical analysis, live session monitoring -- **Capabilities**: Acquisition status, grid processing, real-time data -- **Requirements**: Running SmartEM backend service - -## Claude Code Integration - -The SmartEM MCP server integrates seamlessly with Claude Code, allowing natural language queries about microscopy data directly within your development environment. - -### Prerequisites - -Ensure MCP dependencies are installed (includes FastMCP 2.0): - -```bash -pip install -e .[mcp] -``` - -Verify the installation: - -```bash -python -m smartem_mcp --version -``` - -### Registration with Claude Code - -#### Option 1: Using Claude CLI (Recommended) - -Register the SmartEM MCP server using the Claude CLI: - -```bash -# Basic registration (user-wide scope) -claude mcp add smartem -- python -m smartem_mcp server - -# With custom API URL -claude mcp add smartem -- python -m smartem_mcp server --api-url http://localhost:30080 - -# Project-scoped registration -claude mcp add smartem --scope project -- python -m smartem_mcp server --api-url http://localhost:30080 -``` - -#### Option 2: Manual Configuration - -If you prefer manual configuration, add the server to your Claude Code settings: - -**For user-wide registration:** -Edit `~/.config/claude-code/mcp.json`: - -```json -{ - "mcpServers": { - "smartem": { - "command": "python", - "args": ["-m", "smartem_mcp", "server", "--api-url", "http://localhost:30080"], - "env": { - "SMARTEM_API_URL": "http://localhost:30080", - "SMARTEM_MCP_LOG_LEVEL": "INFO" - } - } - } -} -``` - -**For project-scoped registration:** -Create `.claude/mcp.json` in your project root: - -```json -{ - "mcpServers": { - "smartem": { - "command": "python", - "args": ["-m", "smartem_mcp", "server"], - "env": { - "SMARTEM_API_URL": "http://localhost:30080" - } - } - } -} -``` - -### Registration Options - -#### Environment Variables - -Configure the MCP server behaviour through environment variables: - -- `SMARTEM_API_URL`: Base URL for SmartEM API (default: `http://localhost:30080`) -- `SMARTEM_MCP_LOG_LEVEL`: Logging level (`DEBUG`, `INFO`, `WARNING`, `ERROR`) -- `PYTHON_PATH`: Python executable path if not in system PATH - -Example with custom environment: - -```bash -claude mcp add smartem --env SMARTEM_API_URL=http://production.server:8080 --env SMARTEM_MCP_LOG_LEVEL=DEBUG -- python -m smartem_mcp server -``` - -#### Scope Options - -- **User scope** (default): Available across all Claude Code projects -- **Project scope**: Only available within the current project directory - -```bash -# User-wide (default) -claude mcp add smartem -- python -m smartem_mcp server - -# Project-specific -claude mcp add smartem --scope project -- python -m smartem_mcp server -``` - -### Verification - -#### 1. Check Registration Status - -```bash -# List all registered MCP servers -claude mcp list - -# Check SmartEM server status -claude mcp status smartem -``` - -#### 2. Test Connection - -Start a new Claude Code session and verify the SmartEM tools are available: - -```bash -# In Claude Code, type a natural language query: -# "List all available SmartEM tools" -``` - -You should see the following tools available: -- `parse_epu_directory` - Parse EPU microscopy directories -- `query_quality_metrics` - Find low-quality images and foil holes -- `query_acquisitions` - Query recent acquisition sessions -- `query_grid_status` - Get grid processing status - -**Note**: The client provides high-level methods such as `find_low_quality_items()` which internally call the corresponding MCP tools. - -#### 3. Test Basic Functionality - -Try a simple query in Claude Code: - -> "Parse the EPU directory at /path/to/your/epu/session" - -### Usage in Claude Code - -Once registered, you can interact with SmartEM data using natural language within Claude Code: - -#### Directory Analysis - -> "Show me a comprehensive analysis of the EPU session at /data/microscopy/session_001" - -> "What grids and grid squares are available in /path/to/epu/directory?" - -#### Quality Assessment - -> "Find all images with quality scores below 0.4 in the EPU directory /data/session_001" - -> "Show me foil holes that need attention from /path/to/epu with quality threshold 0.3" - -#### Session Monitoring - -> "What are the 5 most recent microscopy acquisitions?" - -> "Check the processing status of grid uuid-12345" - -#### Advanced Queries - -> "Compare quality metrics between /data/session_001 and /data/session_002" - -> "Generate a report for all low-quality items found today" - -### Server Mode (Standalone) - -For advanced users or debugging, run the MCP server in standalone mode: - -```bash -python -m smartem_mcp server --api-url http://localhost:30080 --log-level DEBUG -``` - -This starts the server with stdio communication, primarily useful for: -- Debugging MCP protocol issues -- Custom client integrations -- Development and testing - -## Available Tools - -### `parse_epu_directory` -Parse EPU microscopy directory and extract comprehensive session data. - -**Parameters:** -- `path` (required): Path to EPU output directory containing EpuSession.dm - -**Returns:** Acquisition data, grids, grid squares, and statistics - -### `query_quality_metrics` -Find foil holes and micrographs with quality scores below threshold. - -**Parameters:** -- `path` (required): Path to EPU directory -- `threshold` (optional): Quality threshold (default: 0.5) -- `source` (optional): "filesystem" or "api" (default: filesystem) - -**Returns:** List of low-quality items with metadata - -### `query_acquisitions` -Query recent microscopy acquisition sessions from API. - -**Parameters:** -- `limit` (optional): Number of acquisitions to return (default: 10) - -**Returns:** List of acquisitions with status and metadata - -### `query_grid_status` -Get detailed status and processing state for specific grid. - -**Parameters:** -- `grid_id` (required): Grid UUID or identifier - -**Returns:** Grid details and processing status - -## Future Enhancements - -The following capabilities are scaffolded for future implementation: - -### Real-time Event Streaming -```python -# Future capability - real-time updates via RabbitMQ -events = await client.subscribe_to_events("acquisition.*.quality_updated") -``` - -### Direct Database Querying -```python -# Future capability - direct read-only database access -result = await client.query_database("SELECT * FROM micrographs WHERE quality < 0.3") -``` - -## Error Handling - -All operations return structured results with success indicators: - -```python -result = await client.parse_epu_directory("/path/to/epu") -if result.get("success"): - # Process result data - data = result.get("data", {}) - pass -else: - print(f"Error: {result.get('error')}") -``` - -## Configuration - -### Environment Variables - -- `SMARTEM_API_URL`: Base URL for SmartEM API (default: http://localhost:30080) -- `SMARTEM_MCP_LOG_LEVEL`: Logging level (default: INFO) - -### API Authentication - -For authenticated API access (future): -```python -client = SmartEMMCPClient(api_token="your_token_here") -``` - -## Troubleshooting - -### Common Issues - -#### 1. MCP Server Registration Problems - -**"SmartEM MCP server not found"** -- Ensure MCP dependencies are installed: `pip install -e .[mcp]` (includes FastMCP 2.0) -- Verify Python path includes smartem_mcp module: `python -c "import smartem_mcp; print('OK')"` -- Check registration with `claude mcp list` - -**"Failed to start MCP server"** -- Verify Python executable is accessible: `which python` -- Test server manually: `python -m smartem_mcp server --api-url http://localhost:30080` -- Check environment variables are properly set - -**"MCP server registered but not responding"** -- Remove and re-add the server: `claude mcp remove smartem && claude mcp add smartem -- python -m smartem_mcp server` -- Restart Claude Code completely -- Check server logs: `claude mcp logs smartem` - -#### 2. Directory and Data Issues - -**"Invalid EPU directory"** -- Verify directory contains EpuSession.dm file -- Check Metadata/ and Images-Disc*/ subdirectories exist -- Ensure proper file permissions for reading EPU data - -**"Permission denied accessing EPU files"** -- Check file permissions: `ls -la /path/to/epu/directory` -- Ensure user has read access to all EPU subdirectories -- For network mounts, verify mount permissions - -#### 3. API Connection Issues - -**"API connection failed"** -- Verify SmartEM backend is running: `curl http://localhost:30080/health` -- Check API URL is correct in registration command -- Test network connectivity to API endpoint - -**"Authentication failed"** (Future feature) -- Verify API tokens are properly configured -- Check token expiration and renewal - -#### 4. Claude Code Integration Issues - -**"Tools not available in Claude Code"** -- Restart Claude Code after registration -- Check MCP server status: `claude mcp status smartem` -- Verify registration scope (user vs project) - -**"Queries return empty results"** -- Test with standalone client: `python -m smartem_mcp client parse --path /path/to/epu` -- Check EPU directory structure and data validity -- Verify API connectivity for acquisition queries - -### Debug Mode - -#### Enable Debug Logging - -For MCP server debugging: -```bash -claude mcp add smartem --env SMARTEM_MCP_LOG_LEVEL=DEBUG -- python -m smartem_mcp server --api-url http://localhost:30080 --log-level DEBUG -``` - -For client debugging: -```bash -python -m smartem_mcp --log-level DEBUG client parse --path /path/to/epu -``` - -#### Check Server Logs - -View MCP server logs in Claude Code: -```bash -claude mcp logs smartem -``` - -#### Manual Server Testing - -Test the server independently of Claude Code: -```bash -# Start server in stdio mode for testing -python -m smartem_mcp server --log-level DEBUG - -# The server uses FastMCP stdio communication protocol -# For testing, use interactive mode or programmatic client instead: -python -m smartem_mcp interactive -``` - -### Environment-Specific Troubleshooting - -#### Development Environment -- Ensure virtual environment is activated when registering -- Use absolute Python paths: `claude mcp add smartem -- /path/to/.venv/bin/python -m smartem_mcp server` - -#### Production Environment -- Verify all dependencies are installed in production Python environment -- Check firewall rules for API connectivity -- Ensure proper logging configuration for production - -#### Container Environments -- Mount EPU directories properly in container -- Expose API ports correctly -- Set environment variables in container runtime - -### Getting Help - -If issues persist: - -1. **Gather diagnostic information:** - ```bash - # System info - python --version - pip list | grep -E "(smartem|mcp)" - claude mcp list - claude mcp status smartem - - # Test basic functionality - python -m smartem_mcp --version - python -c "import smartem_mcp; print('Import successful')" - ``` - -2. **Check project documentation:** [SmartEM Decisions README](/README.md) - -3. **Review MCP protocol documentation:** [Model Context Protocol Specification](https://modelcontextprotocol.io/) - -4. **Submit issue:** Include diagnostic information and specific error messages - -## Integration with Claude Code - -When properly configured, you can ask Claude Code natural language questions: - -> "Show me all grid squares with quality scores below 0.5 from the EPU session at /data/session1" - -> "What's the current status of acquisition abc-123?" - -> "List micrographs acquired in the last hour with defocus values" - -The MCP server will automatically route these queries to the appropriate data sources and return formatted results. diff --git a/webui/src/docs/reference.mdx b/webui/src/docs/reference.mdx deleted file mode 100644 index 19a3600..0000000 --- a/webui/src/docs/reference.mdx +++ /dev/null @@ -1,13 +0,0 @@ -# Reference - -Technical reference material including APIs and release notes. - -## CLI -- [CLI Reference](/docs/reference/cli-reference) - Command-line interface documentation - -## API Documentation -- [Athena Decision Service API](/api/athena/index.html) - Decision service API docs -- [SmartEM Core API](/api/smartem/index.html) - Core API docs - -## External Links -- [Release Notes](https://github.com/DiamondLightSource/smartem-decisions/releases) - GitHub releases diff --git a/webui/src/docs/reference/cli-reference.mdx b/webui/src/docs/reference/cli-reference.mdx deleted file mode 100644 index 17f9a1e..0000000 --- a/webui/src/docs/reference/cli-reference.mdx +++ /dev/null @@ -1,490 +0,0 @@ -# SmartEM Agent CLI Reference - -The SmartEM Agent provides a comprehensive command-line interface for parsing, validating, and monitoring EPU (Electron Physical User) microscopy data. This reference documents all available commands, parameters, and usage patterns. - -## Command Overview - -```bash -python -m smartem_agent [COMMAND] [OPTIONS] -``` - -The CLI is organised into the following command groups: - -- **[parse](#parse-commands)** - Parse various EPU data files and structures -- **[validate](#validate-command)** - Validate EPU project directory structure -- **[watch](#watch-command)** - Monitor directories for real-time data processing - -## Global Options - -All commands support the following verbosity options: - -| Option | Short | Type | Default | Description | -|--------|-------|------|---------|-------------| -| `--verbose` | `-v` | count | 0 | Increase verbosity (-v for INFO, -vv for DEBUG) | - -**Verbosity Levels:** -- **0 (default)**: ERROR level - Only critical errors are shown -- **1 (-v)**: INFO level - General information and warnings -- **2 (-vv)**: DEBUG level - Detailed debugging information - -## Parse Commands - -Parse commands extract and process data from EPU files without persisting to the backend API. These are primarily used for development, debugging, and data validation purposes. - -### parse dir - -Parse an entire EPU output directory structure that may contain multiple grids. - -```bash -python -m smartem_agent parse dir [OPTIONS] EPU_OUTPUT_DIR -``` - -**Parameters:** - -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `epu_output_dir` | str | Yes | Path to EPU output directory containing multiple grid directories | -| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) | - -**Example:** -```bash -python -m smartem_agent parse dir /data/microscopy/session_20240115 -v -``` - -**Use Cases:** -- Batch processing of multiple grid datasets -- Session-level data analysis and validation -- Quality control across entire microscopy sessions - -### parse grid - -Parse a single grid data directory containing EPU acquisition files. - -```bash -python -m smartem_agent parse grid [OPTIONS] GRID_DATA_DIR -``` - -**Parameters:** - -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `grid_data_dir` | str | Yes | Path to individual grid data directory | -| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) | - -**Example:** -```bash -python -m smartem_agent parse grid /data/microscopy/session_20240115/Grid_001 -vv -``` - -**Use Cases:** -- Single grid analysis and debugging -- Grid-specific data extraction -- Verification of individual acquisition datasets - -### parse session - -Parse an EPU session manifest file (typically `EpuSession.dm`). - -```bash -python -m smartem_agent parse session [OPTIONS] PATH -``` - -**Parameters:** - -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `path` | str | Yes | Path to EPU session manifest file | -| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) | - -**Example:** -```bash -python -m smartem_agent parse session /data/microscopy/Grid_001/EpuSession.dm -``` - -**Output:** Session metadata including acquisition parameters, timestamps, and instrument configuration. - -### parse atlas - -Parse an atlas manifest file (typically `Atlas/Atlas.dm`). - -```bash -python -m smartem_agent parse atlas [OPTIONS] PATH -``` - -**Parameters:** - -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `path` | str | Yes | Path to atlas manifest file | -| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) | - -**Example:** -```bash -python -m smartem_agent parse atlas /data/microscopy/Grid_001/Sample*/Atlas/Atlas.dm -``` - -**Output:** Atlas information including grid square positions, tile arrangements, and overview image metadata. - -### parse gridsquare-metadata - -Parse grid square metadata files (typically `Metadata/GridSquare_*.dm`). - -```bash -python -m smartem_agent parse gridsquare-metadata [OPTIONS] PATH -``` - -**Parameters:** - -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `path` | str | Yes | Path to grid square metadata file | -| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) | - -**Example:** -```bash -python -m smartem_agent parse gridsquare-metadata /data/microscopy/Grid_001/Metadata/GridSquare_12345.dm -``` - -**Output:** Grid square-specific metadata including foil hole positions and targeting information. - -### parse gridsquare - -Parse grid square manifest files (typically `Images-Disc*/GridSquare_*/GridSquare_*.xml`). - -```bash -python -m smartem_agent parse gridsquare [OPTIONS] PATH -``` - -**Parameters:** - -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `path` | str | Yes | Path to grid square manifest file | -| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) | - -**Example:** -```bash -python -m smartem_agent parse gridsquare /data/microscopy/Grid_001/Images-Disc1/GridSquare_12345/GridSquare_20240115_143022.xml -``` - -**Output:** Grid square acquisition details and image collection parameters. - -### parse foilhole - -Parse foil hole manifest files (typically `Images-Disc*/GridSquare_*/Data/FoilHole_*.xml`). - -```bash -python -m smartem_agent parse foilhole [OPTIONS] PATH -``` - -**Parameters:** - -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `path` | str | Yes | Path to foil hole manifest file | -| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) | - -**Example:** -```bash -python -m smartem_agent parse foilhole /data/microscopy/Grid_001/Images-Disc1/GridSquare_12345/Data/FoilHole_1234567_Data_20240115_143045_fractions.xml -``` - -**Output:** Foil hole targeting information and acquisition metadata. - -### parse micrograph - -Parse micrograph manifest files (typically `Images-Disc*/GridSquare_*/FoilHoles/FoilHole_*.xml`). - -```bash -python -m smartem_agent parse micrograph [OPTIONS] PATH -``` - -**Parameters:** - -| Parameter | Type | Required | Description | -|-----------|------|----------|-------------| -| `path` | str | Yes | Path to micrograph manifest file | -| `--verbose` | count | No | Verbosity level (see [Global Options](#global-options)) | - -**Example:** -```bash -python -m smartem_agent parse micrograph /data/microscopy/Grid_001/Images-Disc1/GridSquare_12345/FoilHoles/FoilHole_1234567_20240115_143045.xml -``` - -**Output:** Individual micrograph metadata including exposure parameters and image quality metrics. - -## Validate Command - -The validate command checks EPU project directory structure for completeness and compliance with expected formats. - -```bash -python -m smartem_agent validate [OPTIONS] PATH -``` - -**Parameters:** - -| Parameter | Type | Required | Default | Description | -|-----------|------|----------|---------|-------------| -| `path` | str | Yes | - | Path to EPU project directory to validate | -| `--verbose` | count | No | 0 | Verbosity level (see [Global Options](#global-options)) | - -**Exit Codes:** -- **0**: Directory structure is valid -- **1**: Directory structure is invalid (validation errors found) - -**Example:** -```bash -python -m smartem_agent validate /data/microscopy/Grid_001 -v -``` - -**Validation Checks:** -- Presence of required EPU session files -- Directory structure compliance -- File naming convention adherence -- Metadata file accessibility - -**Output:** -- **Valid structure**: Confirmation message with no errors -- **Invalid structure**: Detailed list of structural issues and missing components - -## Watch Command - -The watch command provides real-time monitoring of EPU data directories, automatically processing new files and communicating with the SmartEM backend. - -```bash -python -m smartem_agent watch [OPTIONS] PATH -``` - -### Parameters - -| Parameter | Type | Required | Default | Description | -|-----------|------|----------|---------|-------------| -| `path` | Path | Yes | - | Directory to monitor for file changes | -| `--dry-run` | bool | No | False | Run without making API calls to backend | -| `--api-url` | str | No | `http://127.0.0.1:8000` | Backend API endpoint URL | -| `--log-file` | str | No | `fs_changes.log` | Log file path for change events | -| `--log-interval` | float | No | 10.0 | Logging interval in seconds | -| `--agent-id` | str | No | None | Agent identifier for SSE connection | -| `--session-id` | str | No | None | Session identifier for SSE connection | -| `--sse-timeout` | int | No | 30 | SSE connection timeout in seconds | -| `--heartbeat-interval` | int | No | 60 | Agent heartbeat interval in seconds (0 to disable) | -| `--verbose` | count | No | 0 | Verbosity level (see [Global Options](#global-options)) | - -### Core Parameters - -#### path -**Type:** Path (required) -**Description:** The root directory to monitor for EPU data files. Must be an existing directory containing or expected to contain EPU acquisition data. - -**Example:** -```bash -python -m smartem_agent watch /data/microscopy/active_session -``` - -#### --dry-run -**Type:** Boolean flag -**Default:** False -**Description:** Run the agent in simulation mode without making API calls to the backend. Useful for testing file monitoring and parsing without affecting production systems. - -**Example:** -```bash -python -m smartem_agent watch /data/test --dry-run -``` - -**Behaviour:** -- File monitoring and parsing operate normally -- No data is sent to the backend API -- Uses in-memory data storage only -- Ideal for development and testing scenarios - -### API Integration Parameters - -#### --api-url -**Type:** String -**Default:** `http://127.0.0.1:8000` -**Description:** URL of the SmartEM backend API endpoint. The agent will attempt to connect to this URL for data persistence and real-time communication. - -**Examples:** -```bash -# Local development -python -m smartem_agent watch /data --api-url http://localhost:8000 - -# Production deployment -python -m smartem_agent watch /data --api-url https://smartem.facility.ac.uk/api -``` - -**Connection Validation:** The agent validates API connectivity at startup and exits with error code 1 if the API is unreachable (unless using `--dry-run`). - -#### --agent-id -**Type:** String (optional) -**Description:** Unique identifier for this agent instance. Required for real-time communication with the backend via Server-Sent Events (SSE). - -**Format Recommendations:** -- Use descriptive names: `microscope-01`, `titan-krios-lab2` -- Include facility/location information for multi-site deployments -- Ensure uniqueness across all agent instances - -#### --session-id -**Type:** String (optional) -**Description:** Session identifier linking this agent to a specific microscopy session. Required for real-time communication with the backend. - -**Usage:** Typically generated by the backend or provided by the acquisition software. Used to group related data and instructions. - -### Real-Time Communication Parameters - -#### --sse-timeout -**Type:** Integer -**Default:** 30 seconds -**Description:** Timeout for Server-Sent Events (SSE) connections to the backend. Controls how long the agent waits for backend responses. - -**Recommendations:** -- **Local networks:** 10-30 seconds -- **Remote/unstable connections:** 60-120 seconds -- **High-latency networks:** 120+ seconds - -#### --heartbeat-interval -**Type:** Integer -**Default:** 60 seconds -**Description:** Interval between heartbeat messages sent to the backend to maintain connection health monitoring. Set to 0 to disable heartbeats. - -**Connection Health:** The backend considers connections stale after 2 minutes without heartbeats, so values should be well below 120 seconds. - -**Examples:** -```bash -# Standard monitoring (every minute) -python -m smartem_agent watch /data --agent-id agent-01 --session-id session-123 - -# Frequent monitoring (every 30 seconds) -python -m smartem_agent watch /data --agent-id agent-01 --session-id session-123 --heartbeat-interval 30 - -# Disable heartbeats (not recommended for production) -python -m smartem_agent watch /data --agent-id agent-01 --session-id session-123 --heartbeat-interval 0 -``` - -### Logging Parameters - -#### --log-file -**Type:** String -**Default:** `fs_changes.log` -**Description:** Path to log file for filesystem change events. The agent writes structured JSON logs of all detected file changes to this file. - -**Log Content:** File creation, modification events, timestamps, file sizes, and processing status. - -#### --log-interval -**Type:** Float -**Default:** 10.0 seconds -**Description:** Batching interval for filesystem change logging. Events are collected and written to the log file at this interval to improve performance. - -**Performance Impact:** -- **Lower values (1-5s):** More responsive logging, higher I/O overhead -- **Higher values (30-60s):** Better performance, less frequent updates -- **Very high values (>60s):** Risk of log loss on unexpected shutdown - -### Usage Examples - -#### Basic Monitoring -```bash -# Monitor directory with default settings -python -m smartem_agent watch /data/microscopy/Grid_001 -``` - -#### Production Deployment with Real-Time Communication -```bash -python -m smartem_agent watch /data/microscopy/active_session \ - --api-url https://smartem-backend.facility.ac.uk \ - --agent-id microscope-titan-01 \ - --session-id session-20240115-001 \ - --heartbeat-interval 45 \ - --verbose -``` - -#### Development and Testing -```bash -# Dry run with debug logging -python -m smartem_agent watch /data/test_data \ - --dry-run \ - --log-interval 5.0 \ - --verbose --verbose -``` - -#### High-Performance Production Setup -```bash -python -m smartem_agent watch /data/high_throughput \ - --api-url http://backend:8000 \ - --agent-id facility-workstation-03 \ - --session-id batch-processing-session \ - --log-interval 30.0 \ - --sse-timeout 120 \ - --heartbeat-interval 30 -``` - -## File Pattern Matching - -All commands that process EPU data use consistent file pattern matching to identify relevant files: - -**Supported Patterns:** -- `EpuSession.dm` - EPU session files -- `Metadata/GridSquare_*.dm` - Grid square metadata -- `Images-Disc*/GridSquare_*/GridSquare_*_*.xml` - Grid square manifests -- `Images-Disc*/GridSquare_*/Data/FoilHole_*_Data_*_*_*_*.xml` - Foil hole data -- `Images-Disc*/GridSquare_*/FoilHoles/FoilHole_*_*_*.xml` - Foil hole positions -- `Sample*/Atlas/Atlas.dm` - Atlas files - -**Pattern Matching Rules:** -- Recursive directory traversal for nested structures -- Case-sensitive matching -- Wildcards support standard shell globbing patterns -- Both Windows and Linux path separators are supported - -## Exit Codes - -| Code | Condition | Description | -|------|-----------|-------------| -| 0 | Success | Command completed successfully | -| 1 | Validation Error | Directory validation failed (validate command) | -| 1 | API Connection Error | Unable to connect to backend API | -| 1 | Directory Not Found | Specified directory does not exist | -| 1 | Permission Error | Insufficient permissions to access directory or files | -| 2 | Keyboard Interrupt | User terminated command with Ctrl+C | - -## Integration Notes - -### Backend API Requirements - -The watch command requires a compatible SmartEM backend API with the following endpoints: - -- **Status endpoint**: `GET /status` - For connectivity validation -- **Data persistence**: Various endpoints for storing parsed EPU data -- **SSE endpoint**: `GET /agent/{agent_id}/session/{session_id}/instructions/stream` - For real-time communication -- **Heartbeat endpoint**: `POST /agent/{agent_id}/session/{session_id}/heartbeat` - For connection health - -### Environment Considerations - -**File System Permissions:** -- Read access to EPU data directories -- Write access to log file location -- Appropriate permissions for recursive directory traversal - -**Network Requirements:** -- HTTP/HTTPS connectivity to backend API -- Stable connection for SSE streaming (when using real-time features) -- Consideration of firewall and proxy configurations - -**Resource Requirements:** -- Memory usage scales with directory size and file count -- CPU usage depends on file processing frequency -- Network bandwidth for real-time data transmission - -### Performance Considerations - -**Large Directory Structures:** -- Initial directory parsing can be resource-intensive -- Consider using `--log-interval` tuning for high-frequency file changes -- Monitor memory usage with very large datasets - -**Network Latency:** -- Adjust `--sse-timeout` for high-latency connections -- Consider `--heartbeat-interval` tuning based on network stability -- Use `--dry-run` for local testing and development - -This CLI reference provides comprehensive coverage of all SmartEM Agent command-line functionality. For additional help with specific use cases or troubleshooting, refer to the [CLI Troubleshooting Guide](troubleshoot-cli.md). diff --git a/webui/src/docs/terms-and-defs.mdx b/webui/src/docs/terms-and-defs.mdx deleted file mode 100644 index f5f7c92..0000000 --- a/webui/src/docs/terms-and-defs.mdx +++ /dev/null @@ -1,46 +0,0 @@ -# Terms and Definitions - -## EPU - -Windows desktop application developed by ThermoFisher, serving as the original piece of electron microscopy -acquisition software. EPU (Enhanced Productivity User interface) provides comprehensive control over data collection -parameters and automation workflows for cryo-electron microscopy experiments. - -## SPA - -Single Particle Analysis - A computational method in cryo-electron microscopy used to determine the -three-dimensional structure of biological macromolecules from individual particle images embedded in vitreous ice. - -## Athena / Athena API - -Decision-making software developed by ThermoFisher for automated cryo-electron microscopy workflows. The Athena API -provides programmatic access to session management, decision recording, and algorithm results for optimised data -collection strategies. - -## ARIA - -Automated Real-time Image Analysis system developed by INSTRUCT-ERIC for real-time processing and quality assessment -of cryo-electron microscopy data during acquisition. More information available at: -https://instruct-eric.org/help/about-aria - -## Atlas - -A comprehensive overview image of the entire grid, composed of individual tiles arranged in a 5×5 grid pattern. The -atlas provides spatial context and navigation reference for subsequent high-magnification data collection across the -specimen grid. - -## Grid Square - -Individual sections of the electron microscopy grid that contain the specimen of interest. Grid squares are selected -based on ice quality, specimen distribution, and other quality metrics for detailed imaging at higher magnifications. - -## Foil Hole - -Circular openings in the carbon support film of the electron microscopy grid where the biological specimen is -suspended in vitreous ice. These holes provide areas of minimal background interference for high-resolution imaging of -embedded particles. - -## Micrograph - -High-resolution electron microscopy images captured from individual foil holes or grid squares, containing the -embedded biological specimens used for subsequent image processing and structural analysis. diff --git a/webui/src/docs/tutorials.mdx b/webui/src/docs/tutorials.mdx deleted file mode 100644 index 17a0052..0000000 --- a/webui/src/docs/tutorials.mdx +++ /dev/null @@ -1,5 +0,0 @@ -# Tutorials - -Tutorials for installation and typical usage. New users start here. - -- [Installation](/docs/tutorials/installation) - Get started with SmartEM diff --git a/webui/src/docs/tutorials/installation.mdx b/webui/src/docs/tutorials/installation.mdx deleted file mode 100644 index 77eda4b..0000000 --- a/webui/src/docs/tutorials/installation.mdx +++ /dev/null @@ -1,87 +0,0 @@ -# Installation - -## Check Your Version of Python - -SmartEM Decisions requires Python 3.12 or later for optimal performance and access to modern typing features. You can check your current Python version by entering the following command in a terminal: - -```bash -python3 --version -``` - -**Note:** The project specifically requires Python 3.12+ to utilise advanced typing features and maintain compatibility with the latest scientific computing libraries. - -## Create a Virtual Environment - -It is strongly recommended that you install SmartEM Decisions within a virtual environment to prevent conflicts with existing Python installations and maintain a clean development environment: - -```bash -python3 -m venv /path/to/venv -source /path/to/venv/bin/activate -``` - -For development work, consider creating the virtual environment within the project directory: - -```bash -python3 -m venv .venv -source .venv/bin/activate -``` - -## Installing the Library - -### Standard Installation - -You can install the library and its core dependencies using `pip`: - -```bash -python3 -m pip install smartem-decisions -``` - -### Development Installation - -For development work or if you require the latest features, install directly from the source repository: - -```bash -# Clone and install in development mode -git clone https://github.com/DiamondLightSource/smartem-decisions.git -cd smartem-decisions -pip install -e .[all] -``` - -### Installation with Optional Dependencies - -Install with specific feature sets: - -```bash -# Install with MCP (Model Context Protocol) support -pip install -e .[mcp] - -# Install with all optional dependencies (recommended for development) -pip install -e .[all] -``` - -## Verify Installation - -The library should now be installed with the command-line interface available on your system path. Verify the installation by checking the installed version: - -```bash -smartem-decisions --version -``` - -You can also verify that all components are correctly installed by running: - -```bash -# Test core functionality -python -c "import smartem_backend, smartem_agent, smartem_common; print('Core components imported successfully')" - -# Test MCP functionality (if installed with MCP support) -python -c "import smartem_mcp; print('MCP components available')" -``` - -## Next Steps - -Once installation is complete, you can proceed to: - -- [Run the backend service](../how-to/run-backend.md) -- [Deploy using containers](../how-to/run-container.md) -- [Set up the development environment](../how-to/development-tools.md) -- [Configure logging](../how-to/configure-logging.md) diff --git a/webui/vite.config.ts b/webui/vite.config.ts index 5a0d2b0..3744951 100644 --- a/webui/vite.config.ts +++ b/webui/vite.config.ts @@ -1,3 +1,5 @@ +import { execSync } from 'node:child_process' +import { resolve } from 'node:path' import mdx from '@mdx-js/rollup' import tailwindcss from '@tailwindcss/vite' import { TanStackRouterVite } from '@tanstack/router-vite-plugin' @@ -5,9 +7,69 @@ import react from '@vitejs/plugin-react' import rehypePrettyCode from 'rehype-pretty-code' import remarkFrontmatter from 'remark-frontmatter' import remarkGfm from 'remark-gfm' -import { defineConfig } from 'vite' +import { defineConfig, type Plugin } from 'vite' import tsconfigPaths from 'vite-tsconfig-paths' +/** + * Vite plugin that watches the docs/ directory and re-syncs MDX files on changes. + * This enables hot-reload of documentation during development. + */ +function docsWatchPlugin(): Plugin { + const docsPath = resolve(__dirname, '..', 'docs') + + return { + name: 'docs-watch', + configureServer(server) { + server.watcher.add(docsPath) + + server.watcher.on('change', (file) => { + if (file.startsWith(docsPath) && file.endsWith('.md')) { + console.log(`\n[docs-watch] Detected change in ${file}`) + try { + execSync('npx tsx scripts/generate-mdx-docs.ts', { + cwd: resolve(__dirname), + stdio: 'inherit', + }) + server.ws.send({ type: 'full-reload' }) + } catch (error) { + console.error('[docs-watch] Failed to sync docs:', error) + } + } + }) + + server.watcher.on('add', (file) => { + if (file.startsWith(docsPath) && file.endsWith('.md')) { + console.log(`\n[docs-watch] Detected new file ${file}`) + try { + execSync('npx tsx scripts/generate-mdx-docs.ts', { + cwd: resolve(__dirname), + stdio: 'inherit', + }) + server.ws.send({ type: 'full-reload' }) + } catch (error) { + console.error('[docs-watch] Failed to sync docs:', error) + } + } + }) + + server.watcher.on('unlink', (file) => { + if (file.startsWith(docsPath) && file.endsWith('.md')) { + console.log(`\n[docs-watch] Detected deleted file ${file}`) + try { + execSync('npx tsx scripts/generate-mdx-docs.ts', { + cwd: resolve(__dirname), + stdio: 'inherit', + }) + server.ws.send({ type: 'full-reload' }) + } catch (error) { + console.error('[docs-watch] Failed to sync docs:', error) + } + } + }) + }, + } +} + export default defineConfig({ base: process.env.NODE_ENV === 'production' ? '/smartem-devtools/' : '/', plugins: [ @@ -28,5 +90,6 @@ export default defineConfig({ tailwindcss(), TanStackRouterVite(), tsconfigPaths(), + docsWatchPlugin(), ], }) From a8765fddc0a9bd1933488d7e8533c8f42cadd9bc Mon Sep 17 00:00:00 2001 From: Val Redchenko Date: Fri, 9 Jan 2026 11:19:05 +0000 Subject: [PATCH 2/2] chore: remove Sphinx documentation infrastructure --- .github/workflows/ci.yml | 23 -- README.md | 11 +- claude-code/ARCHITECTURE.md | 2 +- claude-code/CLAUDE.md | 2 +- .../shared/skills/technical-writer/SKILL.md | 43 ++-- .../smartem-decisions/REPO-GUIDELINES.md | 10 +- docs/_api.rst | 16 -- docs/_templates/custom-module-template.rst | 37 --- docs/conf.py | 210 ------------------ docs/genindex.md | 3 - docs/index.md | 83 ------- webui/scripts/generate-mdx-docs.ts | 10 +- 12 files changed, 33 insertions(+), 417 deletions(-) delete mode 100644 docs/_api.rst delete mode 100644 docs/_templates/custom-module-template.rst delete mode 100644 docs/conf.py delete mode 100644 docs/genindex.md delete mode 100644 docs/index.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc4a625..13e0326 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,26 +35,3 @@ jobs: - name: Format check run: npm run format:check - - docs: - name: Documentation Build - runs-on: ubuntu-latest - # TODO: Enable when docs are migrated from smartem-decisions - if: false - - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Setup Python - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - pip install sphinx myst-parser sphinx-copybutton sphinx-design pydata-sphinx-theme - - - name: Build docs - run: sphinx-build -E docs build/html diff --git a/README.md b/README.md index 9fa0d8c..a224bf2 100644 --- a/README.md +++ b/README.md @@ -75,15 +75,16 @@ Opens at http://localhost:5173 ### Documentation -Documentation is built with Sphinx and published to GitHub Pages from smartem-decisions. +Documentation is built as part of the webui and published to GitHub Pages. ```bash -# Build locally (requires sphinx) -cd docs -sphinx-build -E . ../build/html +# Build locally +cd webui +npm install +npm run build ``` -View at: https://diamondlightsource.github.io/smartem-decisions/ +View at: https://diamondlightsource.github.io/smartem-devtools/ ## Claude Code Configuration diff --git a/claude-code/ARCHITECTURE.md b/claude-code/ARCHITECTURE.md index 67a5f39..d2a9331 100644 --- a/claude-code/ARCHITECTURE.md +++ b/claude-code/ARCHITECTURE.md @@ -155,7 +155,7 @@ export ARIA_GQL_LOCAL=http://localhost:9002/graphql | smartem-frontend | Regenerate OpenAPI client | | smartem-agent | Update api_client imports | | fandanGO-cryoem-dls | Update SmartEMAPIClient | -| Docs | Regenerate OpenAPI spec, Sphinx | +| Docs | Regenerate OpenAPI spec | | Containers | Rebuild images | ### When smartem-backend MQ schema changes diff --git a/claude-code/CLAUDE.md b/claude-code/CLAUDE.md index 65c4890..bfba253 100644 --- a/claude-code/CLAUDE.md +++ b/claude-code/CLAUDE.md @@ -128,7 +128,7 @@ Developer tooling, documentation, and workspace configuration for the SmartEM ec - API specs (SmartEM, Athena) **Key directories:** -- `docs/` - Sphinx documentation, ADRs, how-to guides +- `docs/` - Markdown documentation, ADRs, how-to guides (synced to webui as MDX) - `claude-code/` - Skills, repo guidelines, architecture docs - `webui/` - React developer dashboard - `core/` - Repository and workspace config definitions diff --git a/claude-code/shared/skills/technical-writer/SKILL.md b/claude-code/shared/skills/technical-writer/SKILL.md index bdf9eef..6ea51c1 100644 --- a/claude-code/shared/skills/technical-writer/SKILL.md +++ b/claude-code/shared/skills/technical-writer/SKILL.md @@ -185,30 +185,33 @@ Licence information. - Link to detailed documentation - Keep it scannable (headers, bullets, code blocks) -## Sphinx Documentation +## Documentation (smartem-devtools) ### Build Commands ```bash -cd repos/DiamondLightSource/smartem-decisions +cd repos/DiamondLightSource/smartem-devtools/webui -# Full build -sphinx-build -E docs build/html +# Install dependencies +npm install -# Live reload during writing -sphinx-autobuild docs build/html +# Development server with hot reload +npm run dev -# Check links -sphinx-build -b linkcheck docs build/linkcheck +# Production build +npm run build ``` ### Adding New Pages 1. Create `.md` file in appropriate `docs/` subdirectory -2. Add to `toctree` in parent `index.md` -3. Use MyST Markdown syntax +2. Add link to the appropriate navigation file (`docs/how-to.md`, `docs/explanations.md`, etc.) +3. Update `webui/src/docs/navigation.ts` if adding to sidebar +4. Run `npm run prebuild` to sync changes (automatic in dev mode) -### MyST Markdown Features +### Markdown Features + +Documentation uses standard GitHub Flavored Markdown: ```markdown # Headings use standard Markdown @@ -219,19 +222,13 @@ def example(): pass ``` -Admonitions: -```{note} -This is a note. -``` +**Bold**, *italic*, `inline code` -```{warning} -This is a warning. +- Bullet lists +- [Links](/docs/path/to/page) ``` -Cross-references: -{ref}`label-name` -{doc}`path/to/document` -``` +**Note**: Avoid curly braces `{}` in prose as they're interpreted as JSX. Use backticks for values like `` `<10ms` ``. ## Common British vs American Spellings @@ -272,5 +269,5 @@ Before finalising documentation: ## References - ADR format: https://adr.github.io/ -- MyST Markdown: https://myst-parser.readthedocs.io/ -- Sphinx: https://www.sphinx-doc.org/ +- GitHub Flavored Markdown: https://github.github.com/gfm/ +- MDX: https://mdxjs.com/ diff --git a/claude-code/smartem-decisions/REPO-GUIDELINES.md b/claude-code/smartem-decisions/REPO-GUIDELINES.md index 75a746d..c707193 100644 --- a/claude-code/smartem-decisions/REPO-GUIDELINES.md +++ b/claude-code/smartem-decisions/REPO-GUIDELINES.md @@ -56,9 +56,8 @@ ruff format # Pre-commit checks pre-commit run --all-files -# Documentation -sphinx-build -E docs build/html -sphinx-autobuild docs build/html # Live reload +# Documentation (see smartem-devtools webui) +cd ../smartem-devtools/webui && npm run dev # Database migrations python -m alembic upgrade head @@ -97,10 +96,9 @@ python -m alembic revision --autogenerate -m "Description" - **CI/CD**: GitHub Actions with pre-commit hooks ## Documentation -- **Sphinx**: Auto-generated API docs with Napoleon (Google-style docstrings) -- **MyST**: Markdown support in documentation -- **Live development**: Use sphinx-autobuild for real-time doc updates +- **Markdown**: Documentation in `docs/` synced to smartem-devtools webui as MDX - **API documentation**: Swagger/OpenAPI specs auto-generated +- **Live development**: Run `npm run dev` in smartem-devtools/webui for hot-reload ## Available Skills diff --git a/docs/_api.rst b/docs/_api.rst deleted file mode 100644 index f65c812..0000000 --- a/docs/_api.rst +++ /dev/null @@ -1,16 +0,0 @@ -:orphan: - -.. - This page is not included in the TOC tree, but must exist so that the - autosummary pages are generated for smartem_backend and all its - subpackages - -API -=== - -.. autosummary:: - :toctree: _api - :template: custom-module-template.rst - :recursive: - - smartem_backend diff --git a/docs/_templates/custom-module-template.rst b/docs/_templates/custom-module-template.rst deleted file mode 100644 index 9aeca54..0000000 --- a/docs/_templates/custom-module-template.rst +++ /dev/null @@ -1,37 +0,0 @@ -{{ ('``' + fullname + '``') | underline }} - -{%- set filtered_members = [] %} -{%- for item in members %} - {%- if item in functions + classes + exceptions + attributes %} - {% set _ = filtered_members.append(item) %} - {%- endif %} -{%- endfor %} - -.. automodule:: {{ fullname }} - :members: - - {% block modules %} - {% if modules %} - .. rubric:: Submodules - - .. autosummary:: - :toctree: - :template: custom-module-template.rst - :recursive: - {% for item in modules %} - {{ item }} - {%- endfor %} - {% endif %} - {% endblock %} - - {% block members %} - {% if filtered_members %} - .. rubric:: Members - - .. autosummary:: - :nosignatures: - {% for item in filtered_members %} - {{ item }} - {%- endfor %} - {% endif %} - {% endblock %} diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index b81097b..0000000 --- a/docs/conf.py +++ /dev/null @@ -1,210 +0,0 @@ -"""Configuration file for the Sphinx documentation builder. - -This file only contains a selection of the most common options. For a full -list see the documentation: -https://www.sphinx-doc.org/en/master/usage/configuration.html -""" - -import shutil -import sys -from pathlib import Path -from subprocess import check_output - -import requests - -import smartem_backend - -# -- General configuration ------------------------------------------------ - -# General information about the project. -project = "smartem-decisions" - -# The full version, including alpha/beta/rc tags. -release = smartem_backend.__version__ - -# The short X.Y version. -if "+" in release: - # Not on a tag, use branch name - root = Path(__file__).absolute().parent.parent - git_branch = check_output("git branch --show-current".split(), cwd=root) - version = git_branch.decode().strip() -else: - version = release - -extensions = [ - # Use this for generating API docs - "sphinx.ext.autodoc", - # and making summary tables at the top of API docs - "sphinx.ext.autosummary", - # This can parse google style docstrings - "sphinx.ext.napoleon", - # For linking to external sphinx documentation - "sphinx.ext.intersphinx", - # Add links to source code in API docs - "sphinx.ext.viewcode", - # Adds the inheritance-diagram generation directive - "sphinx.ext.inheritance_diagram", - # Add a copy button to each code block - "sphinx_copybutton", - # For the card element - "sphinx_design", - # So we can write markdown files - "myst_parser", -] - -# So we can use the ::: syntax -myst_enable_extensions = ["colon_fence"] - -# If true, Sphinx will warn about all references where the target cannot -# be found. -nitpicky = True - -# A list of (type, target) tuples (by default empty) that should be ignored when -# generating warnings in "nitpicky mode". Note that type should include the -# domain name if present. Example entries would be ('py:func', 'int') or -# ('envvar', 'LD_LIBRARY_PATH'). -nitpick_ignore = [ - ("py:class", "NoneType"), - ("py:class", "'str'"), - ("py:class", "'float'"), - ("py:class", "'int'"), - ("py:class", "'bool'"), - ("py:class", "'object'"), - ("py:class", "'id'"), - ("py:class", "typing_extensions.Literal"), -] - -# Both the class’ and the __init__ method’s docstring are concatenated and -# inserted into the main body of the autoclass directive -autoclass_content = "both" - -# Order the members by the order they appear in the source code -autodoc_member_order = "bysource" - -# Don't inherit docstrings from baseclasses -autodoc_inherit_docstrings = False - -# Document only what is in __all__ -autosummary_ignore_module_all = False - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# Output graphviz directive produced images in a scalable format -graphviz_output_format = "svg" - -# The name of a reST role (builtin or Sphinx extension) to use as the default -# role, that is, for text marked up `like this` -default_role = "any" - -# The master toctree document. -master_doc = "index" - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# These patterns also affect html_static_path and html_extra_path -exclude_patterns = ["_build"] - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# This means you can link things like `str` and `asyncio` to the relevant -# docs in the python documentation. -intersphinx_mapping = {"python": ("https://docs.python.org/3/", None)} - -# A dictionary of graphviz graph attributes for inheritance diagrams. -inheritance_graph_attrs = {"rankdir": "TB"} - -# Ignore localhost links for periodic check that links in docs are valid -linkcheck_ignore = [r"http://localhost:\d+/"] - -# Set copy-button to ignore python and bash prompts -# https://sphinx-copybutton.readthedocs.io/en/latest/use.html#using-regexp-prompt-identifiers -copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: " -copybutton_prompt_is_regexp = True - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = "pydata_sphinx_theme" -github_repo = "smartem-decisions" -github_user = "DiamondLightSource" -switcher_json = f"https://{github_user}.github.io/{github_repo}/switcher.json" -switcher_exists = requests.get(switcher_json).ok -if not switcher_exists: - print( - "*** Can't read version switcher, is GitHub pages enabled? \n" - " Once Docs CI job has successfully run once, set the " - "Github pages source branch to be 'gh-pages' at:\n" - f" https://github.com/{github_user}/{github_repo}/settings/pages", - file=sys.stderr, - ) - -# Theme options for pydata_sphinx_theme -# We don't check switcher because there are 3 possible states for a repo: -# 1. New project, docs are not published so there is no switcher -# 2. Existing project with latest copier template, switcher exists and works -# 3. Existing project with old copier template that makes broken switcher, -# switcher exists but is broken -# Point 3 makes checking switcher difficult, because the updated copier template -# will fix the switcher at the end of the docs workflow, but never gets a chance -# to complete as the docs build warns and fails. -html_theme_options = { - "logo": { - "text": project, - }, - "use_edit_page_button": True, - "github_url": f"https://github.com/{github_user}/{github_repo}", - "icon_links": [ - { - "name": "PyPI", - "url": f"https://pypi.org/project/{project}", - "icon": "fas fa-cube", - } - ], - "switcher": { - "json_url": switcher_json, - "version_match": version, - }, - "check_switcher": False, - "navbar_end": ["theme-switcher", "icon-links", "version-switcher"], - "navigation_with_keys": False, -} - -# A dictionary of values to pass into the template engine’s context for all pages -html_context = { - "github_user": github_user, - "github_repo": github_repo, - "github_version": version, - "doc_path": "docs", -} - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -html_show_sphinx = False - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -html_show_copyright = False - -# Logo -html_logo = "images/dls-logo.svg" -html_favicon = html_logo - - -# Copy API documentation to build output - preserve directory structure -def copy_api_docs(app, exception): - """Copy API documentation preserving directory structure.""" - # Always copy API docs regardless of build status - api_source = Path(app.srcdir) / "api" - api_dest = Path(app.outdir) / "api" - if api_source.exists(): - if api_dest.exists(): - shutil.rmtree(api_dest) - shutil.copytree(api_source, api_dest) - print(f"API docs copied from {api_source} to {api_dest}") - - -def setup(app): - app.connect("build-finished", copy_api_docs) - return {"version": "1.0"} diff --git a/docs/genindex.md b/docs/genindex.md deleted file mode 100644 index 73f1191..0000000 --- a/docs/genindex.md +++ /dev/null @@ -1,3 +0,0 @@ -# Index - - diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index d02585a..0000000 --- a/docs/index.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -html_theme.sidebar_secondary.remove: true ---- - -```{include} ../README.md -:end-before: - -::::{grid} 2 -:gutter: 4 - -:::{grid-item-card} {material-regular}`directions_walk;2em` -```{toctree} -:maxdepth: 2 -tutorials -``` -+++ -Tutorials for installation and typical usage. New users start here. -::: - -:::{grid-item-card} {material-regular}`directions;2em` -```{toctree} -:maxdepth: 2 -how-to -``` -+++ -Practical step-by-step guides for the more experienced user. -::: - -:::{grid-item-card} {material-regular}`info;2em` -```{toctree} -:maxdepth: 2 -explanations -``` -+++ -Explanations of how it works and why it works that way. -::: - -:::{grid-item-card} {material-regular}`menu_book;2em` -```{toctree} -:maxdepth: 2 -reference -``` -+++ -Technical reference material including APIs and release notes. -::: - -:::: - -## API Documentation - -Interactive API documentation is available for all SmartEM Decisions services: - -::::{grid} 2 -:gutter: 4 - -:::{grid-item-card} {material-regular}`psychology;2em` Athena Decision Service API -:link: api/athena/index.html -:link-type: url - -Complete interactive documentation for the Athena Decision Service API, including session management, decision recording, and algorithm results. - -**Features:** Sessions • Decisions • Algorithm Results • Area Management • Configuration -::: - -:::{grid-item-card} {material-regular}`smart_toy;2em` SmartEM Core API -:link: api/smartem/index.html -:link-type: url - -Documentation for the SmartEM Core API for recording electron microscopy sessions and experimental data. - -**Features:** Sessions • Grids • Atlases • Atlas Tiles • Grid Squares • Foil Holes • Micrographs -::: - -:::: diff --git a/webui/scripts/generate-mdx-docs.ts b/webui/scripts/generate-mdx-docs.ts index 55173c1..b903072 100644 --- a/webui/scripts/generate-mdx-docs.ts +++ b/webui/scripts/generate-mdx-docs.ts @@ -8,15 +8,7 @@ * Manual files (index.mdx, navigation.ts) are preserved and not overwritten. */ -import { - existsSync, - mkdirSync, - readdirSync, - readFileSync, - rmSync, - statSync, - writeFileSync, -} from 'node:fs' +import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { dirname, join, relative, resolve } from 'node:path' import { fileURLToPath } from 'node:url'