feat: provision shared database as hidden infrastructure

The shared dcntrlized-db Postgres service is now treated as Maestro
infrastructure: it is hidden from the service grid and auto-started
before any app container when already installed.

- Container model gains role/hidden/capabilities; registry items gain
  hidden; installed metadata persists manifest capabilities so stopped
  services still surface them.
- Registry marks the shared database image as hidden (OCI annotation,
  label, or name fallback).
- Compose label rewrite merges existing service labels instead of
  replacing them, so org.dcntrlized.role/hidden survive.
- New infra helpers ensure the external dcntrlized_net network and
  start the database on the prod path and before each StartContainer.
- Watcher skips manifest fetch for db-role containers and persists
  capabilities from the manifest.
- spec.yaml documents the new fields.
This commit is contained in:
2026-08-10 16:27:58 +01:00
parent d2fb10ca44
commit c1ad85b65c
10 changed files with 622 additions and 38 deletions
+389
View File
@@ -0,0 +1,389 @@
openapi: 3.0.3
info:
title: Dcntrlized Maestro API
version: 1.0.0
description: The API of the Maestro orchestration router. Maestro manages the lifecycle of Dcntrlized service containers and routes inter-service action calls to them.
components:
schemas:
ActionRequest:
type: object
properties:
payload:
type: object
description: Action-specific parameters
ActionResponse:
type: object
properties:
status:
type: string
data:
type: object
SuccessResponse:
type: object
properties:
status:
type: string
example: "success"
message:
type: string
RegistryContainer:
type: object
required: [name, image]
properties:
name:
type: string
example: "keepnote"
version:
type: string
example: "0.0.1"
image:
type: string
example: "git.dcntrlized.org/Dcntrlized/keepnote/keepnote:0.0.1"
digest:
type: string
title:
type: string
example: "KeepNote"
vendor:
type: string
author:
type: string
description:
type: string
source:
type: string
revision:
type: string
licenses:
type: string
url:
type: string
documentation:
type: string
created_at:
type: string
hidden:
type: boolean
description: Hidden infrastructure containers (e.g. the shared dcntrlized-db) are excluded from the service grid.
example: true
PortMapping:
type: object
properties:
host:
type: integer
guest:
type: integer
IPAddress:
type: object
properties:
version:
type: string
enum: [v4, v6]
address:
type: string
Network:
type: object
properties:
name:
type: string
ipAddress:
$ref: '#/components/schemas/IPAddress'
mode:
type: string
enum: [bridge, host, none, overlay, ipvlan, macvlan]
ports:
type: array
items:
$ref: '#/components/schemas/PortMapping'
Container:
type: object
properties:
id:
type: string
name:
type: string
service_name:
type: string
role:
type: string
example: "db"
hidden:
type: boolean
description: Hidden infrastructure containers (e.g. the shared dcntrlized-db) are excluded from the service grid.
capabilities:
type: array
items:
type: string
example: "database"
description: Capabilities declared by the service manifest (e.g. database).
docker_name:
type: string
image:
type: string
version:
type: string
status:
type: string
created_at:
type: string
network:
$ref: '#/components/schemas/Network'
title:
type: string
vendor:
type: string
author:
type: string
description:
type: string
source:
type: string
paths:
/api/actions/{containerId}/{actionName}:
parameters:
- name: actionName
in: path
required: true
description: The name of the specific action to execute
schema:
type: string
- name: containerId
in: path
required: true
description: The ID or name of the target container/service. Accepts a full or partial container ID, a container name, or the service name (matched case-insensitively).
schema:
type: string
get:
summary: Retrieve data or status from a service action
tags: ["actions"]
description: Forwards the request (method, headers, query string, and body) to the target container's /action/{actionName} endpoint. The target service is responsible for authentication; any client-supplied auth headers are forwarded as-is.
responses:
'200':
description: Request successful
content:
application/json:
schema:
$ref: '#/components/schemas/ActionResponse'
'400':
description: Bad Request - Invalid input parameters
'401':
description: Unauthorized - the target service rejected the request
'404':
description: Action or container not found
'500':
description: Failed to forward the request to the container
post:
summary: Execute a state-changing action in the service
tags: ["actions"]
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/ActionRequest'
responses:
'200':
description: Action executed successfully
content:
application/json:
schema:
$ref: '#/components/schemas/ActionResponse'
'400':
description: Bad Request - Invalid input parameters
'401':
description: Unauthorized - the target service rejected the request
'404':
description: Action or container not found
'500':
description: Failed to forward the request to the container
put:
summary: Update configuration for a specific action
tags: ["actions"]
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/ActionRequest'
responses:
'200':
description: Configuration updated
'400':
description: Bad Request - Invalid input parameters
'401':
description: Unauthorized - the target service rejected the request
'404':
description: Action or container not found
'500':
description: Failed to forward the request to the container
delete:
tags: ["actions"]
summary: Reset or remove an action's state
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/ActionRequest'
responses:
'200':
description: Action state reset
'400':
description: Bad Request - Invalid input parameters
'401':
description: Unauthorized - the target service rejected the request
'404':
description: Action or container not found
'500':
description: Failed to forward the request to the container
/api/containers:
get:
summary: List the services Maestro manages
tags: ["containers"]
responses:
'200':
description: List of services
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/Container'
'500':
description: Internal server error
/api/containers/install:
post:
summary: Install a service container (downloads the compose file and pulls the image)
tags: ["containers"]
requestBody:
required: true
content:
application/json:
schema:
$ref: '#/components/schemas/RegistryContainer'
responses:
'200':
description: Container downloaded and image pulled successfully
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessResponse'
'400':
description: Invalid JSON payload or missing required fields (name, image)
'500':
description: Failed to install container
/api/containers/uninstall:
post:
summary: Uninstall a service container and remove it from the cache
tags: ["containers"]
requestBody:
required: true
content:
application/json:
schema:
type: object
required: [name]
properties:
name:
type: string
responses:
'200':
description: Container uninstalled successfully
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessResponse'
'400':
description: Invalid JSON payload or missing required field (name)
'500':
description: Failed to uninstall container
/api/containers/start:
post:
summary: Start a previously installed service container
tags: ["containers"]
requestBody:
required: true
content:
application/json:
schema:
type: object
required: [name]
properties:
name:
type: string
responses:
'200':
description: Container started successfully
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessResponse'
'400':
description: Invalid JSON payload or missing required field (name)
'500':
description: Failed to start container
/api/containers/stop:
post:
summary: Stop a running service container
tags: ["containers"]
requestBody:
required: true
content:
application/json:
schema:
type: object
required: [name]
properties:
name:
type: string
responses:
'200':
description: Container stopped successfully
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessResponse'
'400':
description: Invalid JSON payload or missing required field (name)
'500':
description: Failed to stop container
/api/registry:
get:
summary: List the services available in the Dcntrlized registry
tags: ["registry"]
responses:
'200':
description: List of registry services
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/RegistryContainer'
'500':
description: Internal server error
'502':
description: Failed to connect to upstream registry
/api/ws:
get:
summary: WebSocket endpoint for live container events
tags: ["containers"]
description: Establishes a WebSocket connection that streams container lifecycle events as JSON.
responses:
'101':
description: Switching Protocols - WebSocket upgrade
+12
View File
@@ -213,6 +213,9 @@ func GetRegistry(w http.ResponseWriter, r *http.Request) {
if created, ok := manifest.Annotations["org.opencontainers.image.created"]; ok && created != "" { if created, ok := manifest.Annotations["org.opencontainers.image.created"]; ok && created != "" {
item.CreatedAt = created item.CreatedAt = created
} }
if hidden, ok := manifest.Annotations["org.dcntrlized.hidden"]; ok && hidden == "true" {
item.Hidden = true
}
} }
// 2. Fallback to Image Config Labels (where docker buildx puts them when targeting v2 registries) // 2. Fallback to Image Config Labels (where docker buildx puts them when targeting v2 registries)
@@ -249,6 +252,15 @@ func GetRegistry(w http.ResponseWriter, r *http.Request) {
if item.CreatedAt == "" && labels["org.opencontainers.image.created"] != "" { if item.CreatedAt == "" && labels["org.opencontainers.image.created"] != "" {
item.CreatedAt = labels["org.opencontainers.image.created"] item.CreatedAt = labels["org.opencontainers.image.created"]
} }
if !item.Hidden && labels["org.dcntrlized.hidden"] == "true" {
item.Hidden = true
}
}
// Fallback for infra images published before the hidden label existed:
// the shared database is always treated as hidden infrastructure.
if strings.EqualFold(item.Name, "dcntrlized-db") {
item.Hidden = true
} }
// Resolve the source repository URL: OCI source label if present, else // Resolve the source repository URL: OCI source label if present, else
+15 -8
View File
@@ -71,16 +71,23 @@ func loadComposeProject(ctx context.Context, serviceName string) (*types.Project
// on the project + oneoff labels). Without them, Up() creates containers but the // on the project + oneoff labels). Without them, Up() creates containers but the
// subsequent start() step fails to find them. Also normalize the dcntrlized // subsequent start() step fails to find them. Also normalize the dcntrlized
// service marker to the lowercase project name so label lookups are consistent. // service marker to the lowercase project name so label lookups are consistent.
//
// Labels are merged, never replaced: the compose file may carry its own
// dcntrlized labels (e.g. org.dcntrlized.role=db, org.dcntrlized.hidden=true
// for the shared database) that must survive onto the running container.
for name, s := range project.Services { for name, s := range project.Services {
s.CustomLabels = map[string]string{ labels := types.Labels{}
api.ProjectLabel: project.Name, for k, v := range s.Labels {
api.ServiceLabel: name, labels[k] = v
api.VersionLabel: api.ComposeVersion,
api.WorkingDirLabel: project.WorkingDir,
api.ConfigFilesLabel: strings.Join(project.ComposeFiles, ","),
api.OneoffLabel: "False",
"org.dcntrlized.service": project.Name,
} }
labels[api.ProjectLabel] = project.Name
labels[api.ServiceLabel] = name
labels[api.VersionLabel] = api.ComposeVersion
labels[api.WorkingDirLabel] = project.WorkingDir
labels[api.ConfigFilesLabel] = strings.Join(project.ComposeFiles, ",")
labels[api.OneoffLabel] = "False"
labels["org.dcntrlized.service"] = project.Name
s.CustomLabels = labels
project.Services[name] = s project.Services[name] = s
} }
+97
View File
@@ -0,0 +1,97 @@
// Copyright (C) 2026 Dcntrlized
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
package containers
import (
"context"
"fmt"
"git.dcntrlized.org/Dcntrlized/Maestro/lib/docker"
"git.dcntrlized.org/Dcntrlized/Maestro/lib/logger"
"git.dcntrlized.org/Dcntrlized/Maestro/lib/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/network"
)
const (
// DCNTRLIZEDNetwork is the external Docker network every dcntrlized service joins.
DCNTRLIZEDNetwork = "dcntrlized_net"
// DatabaseServiceName is the shared database service (image, container and DNS name).
DatabaseServiceName = "dcntrlized-db"
)
// EnsureDCNTRLIZEDNetwork creates the external dcntrlized_net network when it is
// missing. Every service compose file declares it as external, so it must exist
// before any compose project is started.
func EnsureDCNTRLIZEDNetwork(ctx context.Context, dc *docker.DockerClient) error {
if _, err := dc.NetworkInspect(ctx, DCNTRLIZEDNetwork, network.InspectOptions{}); err == nil {
return nil
}
if _, err := dc.NetworkCreate(ctx, DCNTRLIZEDNetwork, network.CreateOptions{}); err != nil {
return fmt.Errorf("failed to create network %s: %w", DCNTRLIZEDNetwork, err)
}
logger.Info("Created external Docker network %s", DCNTRLIZEDNetwork)
return nil
}
// EnsureDatabaseRunning ensures the shared database container is running. It
// starts the database only when it is already installed; otherwise it logs and
// leaves installation to the UI. The database is infrastructure, so this must
// never be invoked for the database service itself (StartContainer guards that).
func EnsureDatabaseRunning(ctx context.Context) error {
dc, err := docker.NewDockerClient()
if err != nil {
return fmt.Errorf("failed to initialize docker client: %w", err)
}
defer dc.Close()
if err := EnsureDCNTRLIZEDNetwork(ctx, dc); err != nil {
return err
}
if databaseRunning(ctx, dc) {
return nil
}
if !IsInstalled(DatabaseServiceName) {
logger.Info("Shared database not installed; install %s from the registry to enable database-backed services", DatabaseServiceName)
return nil
}
logger.Info("Starting shared database %s...", DatabaseServiceName)
if err := StartContainer(ctx, types.RegistryContainer{Name: DatabaseServiceName}); err != nil {
return fmt.Errorf("failed to start shared database %s: %w", DatabaseServiceName, err)
}
return nil
}
// databaseRunning reports whether the shared database container is currently up.
func databaseRunning(ctx context.Context, dc *docker.DockerClient) bool {
filter := filters.NewArgs()
filter.Add("label", "org.dcntrlized.service="+DatabaseServiceName)
containers, err := dc.ContainerList(ctx, container.ListOptions{All: true, Filters: filter})
if err != nil {
logger.Warn("Failed to list database containers: %v", err)
return false
}
for _, c := range containers {
if c.State == "running" || c.State == "restarting" {
return true
}
}
return false
}
+34 -11
View File
@@ -59,17 +59,19 @@ func (cache *ContainerCache) LoadInstalledContainers() error {
} }
cache.AddService(types.Container{ cache.AddService(types.Container{
ServiceName: name, ServiceName: name,
Name: metadata.Name, Name: metadata.Name,
Version: metadata.Version, Version: metadata.Version,
Image: metadata.Image, Image: metadata.Image,
Status: "installed", Status: "installed",
CreatedAt: metadata.CreatedAt, CreatedAt: metadata.CreatedAt,
Title: metadata.Title, Hidden: metadata.Hidden,
Vendor: metadata.Vendor, Capabilities: metadata.Capabilities,
Author: metadata.Author, Title: metadata.Title,
Description: metadata.Description, Vendor: metadata.Vendor,
Source: metadata.Source, Author: metadata.Author,
Description: metadata.Description,
Source: metadata.Source,
}) })
logger.Info("Loaded installed container: %s", name) logger.Info("Loaded installed container: %s", name)
} }
@@ -77,6 +79,27 @@ func (cache *ContainerCache) LoadInstalledContainers() error {
return nil return nil
} }
// updateInstalledMetadataCapabilities persists the capabilities reported by a
// service manifest so installed-but-stopped services can still surface them
// (e.g. the database chip) without running.
func updateInstalledMetadataCapabilities(serviceName string, capabilities []string) {
if len(capabilities) == 0 {
return
}
dir, err := getContainerDataDir(serviceName)
if err != nil {
return
}
metadata, err := readInstalledMetadata(dir)
if err != nil {
return
}
metadata.Capabilities = capabilities
if err := writeMetadata(filepath.Join(dir, "metadata.json"), *metadata); err != nil {
logger.Warn("Failed to persist capabilities for %s: %v", serviceName, err)
}
}
// readInstalledMetadata reads and parses the metadata.json of an install dir. // readInstalledMetadata reads and parses the metadata.json of an install dir.
func readInstalledMetadata(dir string) (*types.InstalledMetadata, error) { func readInstalledMetadata(dir string) (*types.InstalledMetadata, error) {
data, err := os.ReadFile(filepath.Join(dir, "metadata.json")) data, err := os.ReadFile(filepath.Join(dir, "metadata.json"))
+12
View File
@@ -42,6 +42,18 @@ func StartContainer(ctx context.Context, item types.RegistryContainer) error {
return err return err
} }
// Infrastructure first: the external dcntrlized_net network must exist
// before any compose project is started, and services that depend on the
// shared database need it running before they boot.
if err := EnsureDCNTRLIZEDNetwork(ctx, dc); err != nil {
return err
}
if serviceName != DatabaseServiceName {
if err := EnsureDatabaseRunning(ctx); err != nil {
logger.Warn("Failed to ensure shared database before starting %s: %v", serviceName, err)
}
}
// The compose file may reference the image by a short name (e.g. // The compose file may reference the image by a short name (e.g.
// "testservice") while install pulled it under its full registry reference. // "testservice") while install pulled it under its full registry reference.
// Make sure those short names exist so Up() can find them. // Make sure those short names exist so Up() can find them.
+40 -19
View File
@@ -89,33 +89,48 @@ func (cache *ContainerCache) WatchContainers(ctx context.Context, dc *docker.Doc
ver, _ := docker.GetContainerVersion(ctx, dc, containerID) ver, _ := docker.GetContainerVersion(ctx, dc, containerID)
// Fetch Manifest (best effort; fall back to basic info if it fails) // Fetch Manifest (best effort; fall back to basic info if it fails)
host := containerID[:12] // Infrastructure containers (e.g. the shared database) have no HTTP
port := event.Actor.Attributes["org.dcntrlized.port"] // manifest endpoint; keep their role marker and skip the fetch.
manifest, err := docker.FetchManifestWithRetry(ctx, host, port, 20, 50*time.Millisecond) role := event.Actor.Attributes["org.dcntrlized.role"]
if err != nil { var manifest *types.Manifest
log.Printf("Error fetching manifest for container %s: %v", containerID, err) if role != "db" {
host := containerID[:12]
port := event.Actor.Attributes["org.dcntrlized.port"]
var fetchErr error
manifest, fetchErr = docker.FetchManifestWithRetry(ctx, host, port, 20, 50*time.Millisecond)
if fetchErr != nil {
log.Printf("Error fetching manifest for container %s: %v", containerID, fetchErr)
}
} }
// Save to Cache // Save to Cache
log.Printf("Service started. Adding container %s - %s to cache", host, containerName) log.Printf("Service started. Adding container %s - %s to cache", containerID[:12], containerName)
existing, _ := cache.services[key] existing, _ := cache.services[key]
if existing.Version != "" && existing.Version != "latest" && (ver == "" || ver == "latest") { if existing.Version != "" && existing.Version != "latest" && (ver == "" || ver == "latest") {
ver = existing.Version ver = existing.Version
} }
capabilities := existing.Capabilities
if manifest != nil {
capabilities = manifest.Capabilities
updateInstalledMetadataCapabilities(serviceLabel, manifest.Capabilities)
}
container := types.Container{ container := types.Container{
ID: containerID, ID: containerID,
ServiceName: serviceLabel, ServiceName: serviceLabel,
DockerName: containerName, DockerName: containerName,
Image: event.Actor.Attributes["image"], Image: event.Actor.Attributes["image"],
CreatedAt: fmt.Sprint(event.TimeNano), CreatedAt: fmt.Sprint(event.TimeNano),
Network: containerNetwork, Network: containerNetwork,
Status: "running", Status: "running",
Version: ver, Version: ver,
Title: existing.Title, Role: role,
Vendor: existing.Vendor, Hidden: existing.Hidden || event.Actor.Attributes["org.dcntrlized.hidden"] == "true",
Author: existing.Author, Capabilities: capabilities,
Description: existing.Description, Title: existing.Title,
Source: existing.Source, Vendor: existing.Vendor,
Author: existing.Author,
Description: existing.Description,
Source: existing.Source,
} }
if manifest != nil { if manifest != nil {
container.Name = manifest.Name container.Name = manifest.Name
@@ -220,6 +235,12 @@ func (cache *ContainerCache) ReconcileContainers(ctx context.Context, dc *docker
entry.Status = normalizeDockerState(c.State) entry.Status = normalizeDockerState(c.State)
entry.ID = c.ID entry.ID = c.ID
entry.DockerName = strings.TrimPrefix(c.Names[0], "/") entry.DockerName = strings.TrimPrefix(c.Names[0], "/")
if role := c.Labels["org.dcntrlized.role"]; role != "" {
entry.Role = role
}
if c.Labels["org.dcntrlized.hidden"] == "true" {
entry.Hidden = true
}
cache.services[key] = entry cache.services[key] = entry
cache.Unlock() cache.Unlock()
} }
+10
View File
@@ -50,6 +50,16 @@ type Container struct {
CreatedAt string `json:"created_at"` CreatedAt string `json:"created_at"`
Network Network `json:"network"` Network Network `json:"network"`
// Role marks infrastructure containers (e.g. "db"); non-infrastructure
// services have an empty role.
Role string `json:"role,omitempty"`
// Hidden marks infrastructure services that Maestro manages but excludes
// from the service grid (e.g. the shared database).
Hidden bool `json:"hidden,omitempty"`
// Capabilities are the service capabilities declared by its manifest
// (e.g. ["database"]), surfaced so the UI can render capability chips.
Capabilities []string `json:"capabilities,omitempty"`
Title string `json:"title,omitempty"` Title string `json:"title,omitempty"`
Vendor string `json:"vendor,omitempty"` Vendor string `json:"vendor,omitempty"`
Author string `json:"author,omitempty"` Author string `json:"author,omitempty"`
+6
View File
@@ -31,6 +31,9 @@ type RegistryContainer struct {
URL string `json:"url,omitempty"` URL string `json:"url,omitempty"`
Documentation string `json:"documentation,omitempty"` Documentation string `json:"documentation,omitempty"`
CreatedAt string `json:"created_at,omitempty"` CreatedAt string `json:"created_at,omitempty"`
// Hidden marks infrastructure images (e.g. the shared database) that are
// managed by Maestro but excluded from the service grid.
Hidden bool `json:"hidden,omitempty"`
} }
type RawPackage struct { type RawPackage struct {
@@ -47,4 +50,7 @@ type RawPackage struct {
type InstalledMetadata struct { type InstalledMetadata struct {
RegistryContainer RegistryContainer
InstalledAt string `json:"installed_at,omitempty"` InstalledAt string `json:"installed_at,omitempty"`
// Capabilities are persisted from the service manifest once the service has
// run, so installed-but-stopped services can still surface them.
Capabilities []string `json:"capabilities,omitempty"`
} }
+7
View File
@@ -55,6 +55,13 @@ func main() {
if err := cache.LoadInstalledContainers(); err != nil { if err := cache.LoadInstalledContainers(); err != nil {
log.Fatalf("Failed to load installed containers: %v", err) log.Fatalf("Failed to load installed containers: %v", err)
} }
// Ensure the shared database is running before reconciling, so its
// container is picked up by the reconciliation below. Failures to start
// the database are not fatal: Maestro keeps running and the UI surfaces
// the database status for manual action.
if err := libcontainers.EnsureDatabaseRunning(ctx); err != nil {
log.Printf("Warning: failed to ensure shared database: %v", err)
}
cache.ReconcileContainers(ctx, dc) cache.ReconcileContainers(ctx, dc)
go cache.WatchContainers(ctx, dc) go cache.WatchContainers(ctx, dc)
} }