feat: provision shared database as hidden infrastructure

The shared dcntrlized-db Postgres service is now treated as Maestro
infrastructure: it is hidden from the service grid and auto-started
before any app container when already installed.

- Container model gains role/hidden/capabilities; registry items gain
  hidden; installed metadata persists manifest capabilities so stopped
  services still surface them.
- Registry marks the shared database image as hidden (OCI annotation,
  label, or name fallback).
- Compose label rewrite merges existing service labels instead of
  replacing them, so org.dcntrlized.role/hidden survive.
- New infra helpers ensure the external dcntrlized_net network and
  start the database on the prod path and before each StartContainer.
- Watcher skips manifest fetch for db-role containers and persists
  capabilities from the manifest.
- spec.yaml documents the new fields.

spec.yaml

@@ -0,0 +1,389 @@
+openapi: 3.0.3
+info:
+  title: Dcntrlized Maestro API
+  version: 1.0.0
+  description: The API of the Maestro orchestration router. Maestro manages the lifecycle of Dcntrlized service containers and routes inter-service action calls to them.
+
+components:
+  schemas:
+    ActionRequest:
+      type: object
+      properties:
+        payload:
+          type: object
+          description: Action-specific parameters
+    ActionResponse:
+      type: object
+      properties:
+        status:
+          type: string
+        data:
+          type: object
+
+    SuccessResponse:
+      type: object
+      properties:
+        status:
+          type: string
+          example: "success"
+        message:
+          type: string
+
+    RegistryContainer:
+      type: object
+      required: [name, image]
+      properties:
+        name:
+          type: string
+          example: "keepnote"
+        version:
+          type: string
+          example: "0.0.1"
+        image:
+          type: string
+          example: "git.dcntrlized.org/Dcntrlized/keepnote/keepnote:0.0.1"
+        digest:
+          type: string
+        title:
+          type: string
+          example: "KeepNote"
+        vendor:
+          type: string
+        author:
+          type: string
+        description:
+          type: string
+        source:
+          type: string
+        revision:
+          type: string
+        licenses:
+          type: string
+        url:
+          type: string
+        documentation:
+          type: string
+        created_at:
+          type: string
+        hidden:
+          type: boolean
+          description: Hidden infrastructure containers (e.g. the shared dcntrlized-db) are excluded from the service grid.
+          example: true
+
+    PortMapping:
+      type: object
+      properties:
+        host:
+          type: integer
+        guest:
+          type: integer
+
+    IPAddress:
+      type: object
+      properties:
+        version:
+          type: string
+          enum: [v4, v6]
+        address:
+          type: string
+
+    Network:
+      type: object
+      properties:
+        name:
+          type: string
+        ipAddress:
+          $ref: '#/components/schemas/IPAddress'
+        mode:
+          type: string
+          enum: [bridge, host, none, overlay, ipvlan, macvlan]
+        ports:
+          type: array
+          items:
+            $ref: '#/components/schemas/PortMapping'
+
+    Container:
+      type: object
+      properties:
+        id:
+          type: string
+        name:
+          type: string
+        service_name:
+          type: string
+        role:
+          type: string
+          example: "db"
+        hidden:
+          type: boolean
+          description: Hidden infrastructure containers (e.g. the shared dcntrlized-db) are excluded from the service grid.
+        capabilities:
+          type: array
+          items:
+            type: string
+            example: "database"
+          description: Capabilities declared by the service manifest (e.g. database).
+        docker_name:
+          type: string
+        image:
+          type: string
+        version:
+          type: string
+        status:
+          type: string
+        created_at:
+          type: string
+        network:
+          $ref: '#/components/schemas/Network'
+        title:
+          type: string
+        vendor:
+          type: string
+        author:
+          type: string
+        description:
+          type: string
+        source:
+          type: string
+
+paths:
+  /api/actions/{containerId}/{actionName}:
+    parameters:
+      - name: actionName
+        in: path
+        required: true
+        description: The name of the specific action to execute
+        schema:
+          type: string
+      - name: containerId
+        in: path
+        required: true
+        description: The ID or name of the target container/service. Accepts a full or partial container ID, a container name, or the service name (matched case-insensitively).
+        schema:
+          type: string
+    get:
+      summary: Retrieve data or status from a service action
+      tags: ["actions"]
+      description: Forwards the request (method, headers, query string, and body) to the target container's /action/{actionName} endpoint. The target service is responsible for authentication; any client-supplied auth headers are forwarded as-is.
+      responses:
+        '200':
+          description: Request successful
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ActionResponse'
+        '400':
+          description: Bad Request - Invalid input parameters
+        '401':
+          description: Unauthorized - the target service rejected the request
+        '404':
+          description: Action or container not found
+        '500':
+          description: Failed to forward the request to the container
+    post:
+      summary: Execute a state-changing action in the service
+      tags: ["actions"]
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ActionRequest'
+      responses:
+        '200':
+          description: Action executed successfully
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/ActionResponse'
+        '400':
+          description: Bad Request - Invalid input parameters
+        '401':
+          description: Unauthorized - the target service rejected the request
+        '404':
+          description: Action or container not found
+        '500':
+          description: Failed to forward the request to the container
+    put:
+      summary: Update configuration for a specific action
+      tags: ["actions"]
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ActionRequest'
+      responses:
+        '200':
+          description: Configuration updated
+        '400':
+          description: Bad Request - Invalid input parameters
+        '401':
+          description: Unauthorized - the target service rejected the request
+        '404':
+          description: Action or container not found
+        '500':
+          description: Failed to forward the request to the container
+    delete:
+      tags: ["actions"]
+      summary: Reset or remove an action's state
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/ActionRequest'
+      responses:
+        '200':
+          description: Action state reset
+        '400':
+          description: Bad Request - Invalid input parameters
+        '401':
+          description: Unauthorized - the target service rejected the request
+        '404':
+          description: Action or container not found
+        '500':
+          description: Failed to forward the request to the container
+
+  /api/containers:
+    get:
+      summary: List the services Maestro manages
+      tags: ["containers"]
+      responses:
+        '200':
+          description: List of services
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/Container'
+        '500':
+          description: Internal server error
+
+  /api/containers/install:
+    post:
+      summary: Install a service container (downloads the compose file and pulls the image)
+      tags: ["containers"]
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/RegistryContainer'
+      responses:
+        '200':
+          description: Container downloaded and image pulled successfully
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessResponse'
+        '400':
+          description: Invalid JSON payload or missing required fields (name, image)
+        '500':
+          description: Failed to install container
+
+  /api/containers/uninstall:
+    post:
+      summary: Uninstall a service container and remove it from the cache
+      tags: ["containers"]
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required: [name]
+              properties:
+                name:
+                  type: string
+      responses:
+        '200':
+          description: Container uninstalled successfully
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessResponse'
+        '400':
+          description: Invalid JSON payload or missing required field (name)
+        '500':
+          description: Failed to uninstall container
+
+  /api/containers/start:
+    post:
+      summary: Start a previously installed service container
+      tags: ["containers"]
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required: [name]
+              properties:
+                name:
+                  type: string
+      responses:
+        '200':
+          description: Container started successfully
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessResponse'
+        '400':
+          description: Invalid JSON payload or missing required field (name)
+        '500':
+          description: Failed to start container
+
+  /api/containers/stop:
+    post:
+      summary: Stop a running service container
+      tags: ["containers"]
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required: [name]
+              properties:
+                name:
+                  type: string
+      responses:
+        '200':
+          description: Container stopped successfully
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/SuccessResponse'
+        '400':
+          description: Invalid JSON payload or missing required field (name)
+        '500':
+          description: Failed to stop container
+
+  /api/registry:
+    get:
+      summary: List the services available in the Dcntrlized registry
+      tags: ["registry"]
+      responses:
+        '200':
+          description: List of registry services
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  $ref: '#/components/schemas/RegistryContainer'
+        '500':
+          description: Internal server error
+        '502':
+          description: Failed to connect to upstream registry
+
+  /api/ws:
+    get:
+      summary: WebSocket endpoint for live container events
+      tags: ["containers"]
+      description: Establishes a WebSocket connection that streams container lifecycle events as JSON.
+      responses:
+        '101':
+          description: Switching Protocols - WebSocket upgrade

src/handlers/registry/registry.go

@@ -213,6 +213,9 @@ func GetRegistry(w http.ResponseWriter, r *http.Request) {
 			if created, ok := manifest.Annotations["org.opencontainers.image.created"]; ok && created != "" {
 				item.CreatedAt = created
 			}
+			if hidden, ok := manifest.Annotations["org.dcntrlized.hidden"]; ok && hidden == "true" {
+				item.Hidden = true
+			}
 		}
 
 		// 2. Fallback to Image Config Labels (where docker buildx puts them when targeting v2 registries)
@@ -249,6 +252,15 @@ func GetRegistry(w http.ResponseWriter, r *http.Request) {
 			if item.CreatedAt == "" && labels["org.opencontainers.image.created"] != "" {
 				item.CreatedAt = labels["org.opencontainers.image.created"]
 			}
+			if !item.Hidden && labels["org.dcntrlized.hidden"] == "true" {
+				item.Hidden = true
+			}
+		}
+
+		// Fallback for infra images published before the hidden label existed:
+		// the shared database is always treated as hidden infrastructure.
+		if strings.EqualFold(item.Name, "dcntrlized-db") {
+			item.Hidden = true
 		}
 
 		// Resolve the source repository URL: OCI source label if present, else

src/lib/containers/compose.go

@@ -71,16 +71,23 @@ func loadComposeProject(ctx context.Context, serviceName string) (*types.Project
 	// on the project + oneoff labels). Without them, Up() creates containers but the
 	// subsequent start() step fails to find them. Also normalize the dcntrlized
 	// service marker to the lowercase project name so label lookups are consistent.
+	//
+	// Labels are merged, never replaced: the compose file may carry its own
+	// dcntrlized labels (e.g. org.dcntrlized.role=db, org.dcntrlized.hidden=true
+	// for the shared database) that must survive onto the running container.
 	for name, s := range project.Services {
-		s.CustomLabels = map[string]string{
-			api.ProjectLabel:          project.Name,
-			api.ServiceLabel:          name,
-			api.VersionLabel:          api.ComposeVersion,
-			api.WorkingDirLabel:       project.WorkingDir,
-			api.ConfigFilesLabel:      strings.Join(project.ComposeFiles, ","),
-			api.OneoffLabel:           "False",
-			"org.dcntrlized.service":  project.Name,
+		labels := types.Labels{}
+		for k, v := range s.Labels {
+			labels[k] = v
 		}
+		labels[api.ProjectLabel] = project.Name
+		labels[api.ServiceLabel] = name
+		labels[api.VersionLabel] = api.ComposeVersion
+		labels[api.WorkingDirLabel] = project.WorkingDir
+		labels[api.ConfigFilesLabel] = strings.Join(project.ComposeFiles, ",")
+		labels[api.OneoffLabel] = "False"
+		labels["org.dcntrlized.service"] = project.Name
+		s.CustomLabels = labels
 		project.Services[name] = s
 	}
 

src/lib/containers/infra.go

@@ -0,0 +1,97 @@
+// Copyright (C) 2026 Dcntrlized
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package containers
+
+import (
+	"context"
+	"fmt"
+
+	"git.dcntrlized.org/Dcntrlized/Maestro/lib/docker"
+	"git.dcntrlized.org/Dcntrlized/Maestro/lib/logger"
+	"git.dcntrlized.org/Dcntrlized/Maestro/lib/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/api/types/network"
+)
+
+const (
+	// DCNTRLIZEDNetwork is the external Docker network every dcntrlized service joins.
+	DCNTRLIZEDNetwork = "dcntrlized_net"
+	// DatabaseServiceName is the shared database service (image, container and DNS name).
+	DatabaseServiceName = "dcntrlized-db"
+)
+
+// EnsureDCNTRLIZEDNetwork creates the external dcntrlized_net network when it is
+// missing. Every service compose file declares it as external, so it must exist
+// before any compose project is started.
+func EnsureDCNTRLIZEDNetwork(ctx context.Context, dc *docker.DockerClient) error {
+	if _, err := dc.NetworkInspect(ctx, DCNTRLIZEDNetwork, network.InspectOptions{}); err == nil {
+		return nil
+	}
+	if _, err := dc.NetworkCreate(ctx, DCNTRLIZEDNetwork, network.CreateOptions{}); err != nil {
+		return fmt.Errorf("failed to create network %s: %w", DCNTRLIZEDNetwork, err)
+	}
+	logger.Info("Created external Docker network %s", DCNTRLIZEDNetwork)
+	return nil
+}
+
+// EnsureDatabaseRunning ensures the shared database container is running. It
+// starts the database only when it is already installed; otherwise it logs and
+// leaves installation to the UI. The database is infrastructure, so this must
+// never be invoked for the database service itself (StartContainer guards that).
+func EnsureDatabaseRunning(ctx context.Context) error {
+	dc, err := docker.NewDockerClient()
+	if err != nil {
+		return fmt.Errorf("failed to initialize docker client: %w", err)
+	}
+	defer dc.Close()
+
+	if err := EnsureDCNTRLIZEDNetwork(ctx, dc); err != nil {
+		return err
+	}
+
+	if databaseRunning(ctx, dc) {
+		return nil
+	}
+	if !IsInstalled(DatabaseServiceName) {
+		logger.Info("Shared database not installed; install %s from the registry to enable database-backed services", DatabaseServiceName)
+		return nil
+	}
+
+	logger.Info("Starting shared database %s...", DatabaseServiceName)
+	if err := StartContainer(ctx, types.RegistryContainer{Name: DatabaseServiceName}); err != nil {
+		return fmt.Errorf("failed to start shared database %s: %w", DatabaseServiceName, err)
+	}
+	return nil
+}
+
+// databaseRunning reports whether the shared database container is currently up.
+func databaseRunning(ctx context.Context, dc *docker.DockerClient) bool {
+	filter := filters.NewArgs()
+	filter.Add("label", "org.dcntrlized.service="+DatabaseServiceName)
+
+	containers, err := dc.ContainerList(ctx, container.ListOptions{All: true, Filters: filter})
+	if err != nil {
+		logger.Warn("Failed to list database containers: %v", err)
+		return false
+	}
+	for _, c := range containers {
+		if c.State == "running" || c.State == "restarting" {
+			return true
+		}
+	}
+	return false
+}

src/lib/containers/installed.go

@@ -65,6 +65,8 @@ func (cache *ContainerCache) LoadInstalledContainers() error {
 			Image:        metadata.Image,
 			Status:       "installed",
 			CreatedAt:    metadata.CreatedAt,
+			Hidden:       metadata.Hidden,
+			Capabilities: metadata.Capabilities,
 			Title:        metadata.Title,
 			Vendor:       metadata.Vendor,
 			Author:       metadata.Author,
@@ -77,6 +79,27 @@ func (cache *ContainerCache) LoadInstalledContainers() error {
 	return nil
 }
 
+// updateInstalledMetadataCapabilities persists the capabilities reported by a
+// service manifest so installed-but-stopped services can still surface them
+// (e.g. the database chip) without running.
+func updateInstalledMetadataCapabilities(serviceName string, capabilities []string) {
+	if len(capabilities) == 0 {
+		return
+	}
+	dir, err := getContainerDataDir(serviceName)
+	if err != nil {
+		return
+	}
+	metadata, err := readInstalledMetadata(dir)
+	if err != nil {
+		return
+	}
+	metadata.Capabilities = capabilities
+	if err := writeMetadata(filepath.Join(dir, "metadata.json"), *metadata); err != nil {
+		logger.Warn("Failed to persist capabilities for %s: %v", serviceName, err)
+	}
+}
+
 // readInstalledMetadata reads and parses the metadata.json of an install dir.
 func readInstalledMetadata(dir string) (*types.InstalledMetadata, error) {
 	data, err := os.ReadFile(filepath.Join(dir, "metadata.json"))

src/lib/containers/start.go

@@ -42,6 +42,18 @@ func StartContainer(ctx context.Context, item types.RegistryContainer) error {
 		return err
 	}
 
+	// Infrastructure first: the external dcntrlized_net network must exist
+	// before any compose project is started, and services that depend on the
+	// shared database need it running before they boot.
+	if err := EnsureDCNTRLIZEDNetwork(ctx, dc); err != nil {
+		return err
+	}
+	if serviceName != DatabaseServiceName {
+		if err := EnsureDatabaseRunning(ctx); err != nil {
+			logger.Warn("Failed to ensure shared database before starting %s: %v", serviceName, err)
+		}
+	}
+
 	// The compose file may reference the image by a short name (e.g.
 	// "testservice") while install pulled it under its full registry reference.
 	// Make sure those short names exist so Up() can find them.

src/lib/containers/watcher.go

@@ -89,19 +89,31 @@ func (cache *ContainerCache) WatchContainers(ctx context.Context, dc *docker.Doc
 				ver, _ := docker.GetContainerVersion(ctx, dc, containerID)
 
 				// Fetch Manifest (best effort; fall back to basic info if it fails)
+				// Infrastructure containers (e.g. the shared database) have no HTTP
+				// manifest endpoint; keep their role marker and skip the fetch.
+				role := event.Actor.Attributes["org.dcntrlized.role"]
+				var manifest *types.Manifest
+				if role != "db" {
 					host := containerID[:12]
 					port := event.Actor.Attributes["org.dcntrlized.port"]
-				manifest, err := docker.FetchManifestWithRetry(ctx, host, port, 20, 50*time.Millisecond)
-				if err != nil {
-					log.Printf("Error fetching manifest for container %s: %v", containerID, err)
+					var fetchErr error
+					manifest, fetchErr = docker.FetchManifestWithRetry(ctx, host, port, 20, 50*time.Millisecond)
+					if fetchErr != nil {
+						log.Printf("Error fetching manifest for container %s: %v", containerID, fetchErr)
+					}
 				}
 
 				// Save to Cache
-				log.Printf("Service started. Adding container %s - %s to cache", host, containerName)
+				log.Printf("Service started. Adding container %s - %s to cache", containerID[:12], containerName)
 				existing, _ := cache.services[key]
 				if existing.Version != "" && existing.Version != "latest" && (ver == "" || ver == "latest") {
 					ver = existing.Version
 				}
+				capabilities := existing.Capabilities
+				if manifest != nil {
+					capabilities = manifest.Capabilities
+					updateInstalledMetadataCapabilities(serviceLabel, manifest.Capabilities)
+				}
 				container := types.Container{
 					ID:           containerID,
 					ServiceName:  serviceLabel,
@@ -111,6 +123,9 @@ func (cache *ContainerCache) WatchContainers(ctx context.Context, dc *docker.Doc
 					Network:      containerNetwork,
 					Status:       "running",
 					Version:      ver,
+					Role:         role,
+					Hidden:       existing.Hidden || event.Actor.Attributes["org.dcntrlized.hidden"] == "true",
+					Capabilities: capabilities,
 					Title:        existing.Title,
 					Vendor:       existing.Vendor,
 					Author:       existing.Author,
@@ -220,6 +235,12 @@ func (cache *ContainerCache) ReconcileContainers(ctx context.Context, dc *docker
 		entry.Status = normalizeDockerState(c.State)
 		entry.ID = c.ID
 		entry.DockerName = strings.TrimPrefix(c.Names[0], "/")
+		if role := c.Labels["org.dcntrlized.role"]; role != "" {
+			entry.Role = role
+		}
+		if c.Labels["org.dcntrlized.hidden"] == "true" {
+			entry.Hidden = true
+		}
 		cache.services[key] = entry
 		cache.Unlock()
 	}

src/lib/types/docker.go

@@ -50,6 +50,16 @@ type Container struct {
 	CreatedAt   string  `json:"created_at"`
 	Network     Network `json:"network"`
 
+	// Role marks infrastructure containers (e.g. "db"); non-infrastructure
+	// services have an empty role.
+	Role string `json:"role,omitempty"`
+	// Hidden marks infrastructure services that Maestro manages but excludes
+	// from the service grid (e.g. the shared database).
+	Hidden bool `json:"hidden,omitempty"`
+	// Capabilities are the service capabilities declared by its manifest
+	// (e.g. ["database"]), surfaced so the UI can render capability chips.
+	Capabilities []string `json:"capabilities,omitempty"`
+
 	Title       string `json:"title,omitempty"`
 	Vendor      string `json:"vendor,omitempty"`
 	Author      string `json:"author,omitempty"`

src/lib/types/registry.go

@@ -31,6 +31,9 @@ type RegistryContainer struct {
 	URL           string `json:"url,omitempty"`
 	Documentation string `json:"documentation,omitempty"`
 	CreatedAt     string `json:"created_at,omitempty"`
+	// Hidden marks infrastructure images (e.g. the shared database) that are
+	// managed by Maestro but excluded from the service grid.
+	Hidden bool `json:"hidden,omitempty"`
 }
 
 type RawPackage struct {
@@ -47,4 +50,7 @@ type RawPackage struct {
 type InstalledMetadata struct {
 	RegistryContainer
 	InstalledAt string `json:"installed_at,omitempty"`
+	// Capabilities are persisted from the service manifest once the service has
+	// run, so installed-but-stopped services can still surface them.
+	Capabilities []string `json:"capabilities,omitempty"`
 }

src/main.go

@@ -55,6 +55,13 @@ func main() {
 		if err := cache.LoadInstalledContainers(); err != nil {
 			log.Fatalf("Failed to load installed containers: %v", err)
 		}
+		// Ensure the shared database is running before reconciling, so its
+		// container is picked up by the reconciliation below. Failures to start
+		// the database are not fatal: Maestro keeps running and the UI surfaces
+		// the database status for manual action.
+		if err := libcontainers.EnsureDatabaseRunning(ctx); err != nil {
+			log.Printf("Warning: failed to ensure shared database: %v", err)
+		}
 		cache.ReconcileContainers(ctx, dc)
 		go cache.WatchContainers(ctx, dc)
 	}