feat(infra): auto-install and start shared database/auth at boot when missing

ensureInfraRunning now resolves the missing infra service from the registry (via the new lib/registry package, extracted from the GetRegistry handler so the UI and the bootstrap share one source of truth) and installs it before starting it. A fresh host boots with its database and auth service without manual UI steps; failures stay non-fatal warnings like before.
2026-08-16 03:05:30 +01:00
parent 92f4787146
commit 483ed3b531
4 changed files with 429 additions and 312 deletions
@@ -21,6 +21,7 @@ import (

 	"git.dcntrlized.org/Dcntrlized/Maestro/lib/docker"
 	"git.dcntrlized.org/Dcntrlized/Maestro/lib/logger"
+	libregistry "git.dcntrlized.org/Dcntrlized/Maestro/lib/registry"
 	"git.dcntrlized.org/Dcntrlized/Maestro/lib/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/filters"
@@ -50,10 +51,11 @@ func EnsureDCNTRLIZEDNetwork(ctx context.Context, dc *docker.DockerClient) error
 	return nil
 }

-// EnsureDatabaseRunning ensures the shared database container is running. It
-// starts the database only when it is already installed; otherwise it logs and
-// leaves installation to the UI. The database is infrastructure, so this must
-// never be invoked for the database service itself (StartContainer guards that).
+// EnsureDatabaseRunning ensures the shared database container is running. On a
+// fresh host — when the database is not installed yet — it resolves the
+// database from the registry and installs it automatically before starting it.
+// The database is infrastructure, so this must never be invoked for the
+// database service itself (StartContainer guards that).
 func EnsureDatabaseRunning(ctx context.Context) error {
 	return ensureInfraRunning(ctx, DatabaseServiceName, "shared database")
 }
@@ -61,14 +63,17 @@ func EnsureDatabaseRunning(ctx context.Context) error {
 // EnsureAuthRunning ensures the shared auth service container is running. The
 // auth service is hidden infrastructure: every consumer proxies /auth/* to it
 // and validates action tokens with its key, so it must be up before services
-// that depend on it start. Only starts when already installed.
+// that depend on it start. Like the database, it is installed automatically
+// from the registry when it is missing.
 func EnsureAuthRunning(ctx context.Context) error {
 	return ensureInfraRunning(ctx, AuthServiceName, "shared auth service")
 }

-// ensureInfraRunning starts a hidden infrastructure container (shared database,
-// shared auth service) when it is installed but not running. It never installs
-// infrastructure itself — the UI does that from the registry.
+// ensureInfraRunning makes sure a hidden infrastructure container (shared
+// database, shared auth service) is running. When the service is not installed
+// yet it resolves the registry item and installs it first, so a fresh host
+// boots with its infrastructure without manual UI steps. Failures are returned
+// to the caller, which treats them as non-fatal warnings.
 func ensureInfraRunning(ctx context.Context, serviceName, label string) error {
 	dc, err := docker.NewDockerClient()
 	if err != nil {
@@ -84,8 +89,14 @@ func ensureInfraRunning(ctx context.Context, serviceName, label string) error {
 		return nil
 	}
 	if !IsInstalled(serviceName) {
-		logger.Info("%s not installed; install %s from the registry to enable services that need it", label, serviceName)
-		return nil
+		logger.Info("%s %s not installed; installing from the registry...", label, serviceName)
+		item, err := libregistry.ItemByName(ctx, serviceName)
+		if err != nil {
+			return fmt.Errorf("failed to resolve %s %s from the registry: %w", label, serviceName, err)
+		}
+		if err := InstallContainer(ctx, *item); err != nil {
+			return fmt.Errorf("failed to install %s %s: %w", label, serviceName, err)
+		}
 	}

 	logger.Info("Starting %s %s...", label, serviceName)
@@ -0,0 +1,349 @@
+// Copyright (C) 2026 Dcntrlized
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+// Package registry resolves services from the Dcntrlized container registry
+// (a Gitea package registry) into types.RegistryContainer items. It is the
+// single source of truth behind the registry UI handler and the
+// infrastructure bootstrap (lib/containers.ensureInfraRunning), so a shared
+// database that Maestro auto-installs resolves to the exact same image and
+// compose repository the UI would install.
+package registry
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"git.dcntrlized.org/Dcntrlized/Maestro/lib/logger"
+	"git.dcntrlized.org/Dcntrlized/Maestro/lib/types"
+
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"golang.org/x/mod/semver"
+)
+
+// BaseURL returns the Gitea instance base URL (scheme://host[:port]).
+// Production defaults to git.dcntrlized.org; dev mode (DCNTRLIZED_DEV=true)
+// overrides it with GITEA_URL when set.
+func BaseURL() string {
+	if os.Getenv("DCNTRLIZED_DEV") == "true" {
+		if u := strings.TrimRight(os.Getenv("GITEA_URL"), "/"); u != "" {
+			return u
+		}
+	}
+	return "https://git.dcntrlized.org"
+}
+
+// RegistryHost strips the scheme from the Gitea base URL, returning just the
+// host[:port] used in container image references.
+func RegistryHost() string {
+	base := BaseURL()
+	if i := strings.Index(base, "://"); i >= 0 {
+		return base[i+3:]
+	}
+	return base
+}
+
+// Scheme returns the scheme (http/https) of the configured Gitea base URL.
+func Scheme() string {
+	if i := strings.Index(BaseURL(), "://"); i >= 0 {
+		return BaseURL()[:i]
+	}
+	return "https"
+}
+
+// IsInsecure reports whether the Gitea registry is served over plain HTTP, in
+// which case image references must be parsed with name.Insecure.
+func IsInsecure() bool {
+	return strings.HasPrefix(BaseURL(), "http://")
+}
+
+// RepoURLFromImageRef derives the repository browse URL from a container image
+// reference of the form host/org/repo:tag (scheme://host/org/repo), using the
+// scheme of the configured Gitea base URL.
+func RepoURLFromImageRef(imageRef string) string {
+	ref, err := name.ParseReference(imageRef)
+	if err != nil {
+		return ""
+	}
+	return Scheme() + "://" + ref.Context().RegistryStr() + "/" + ref.Context().RepositoryStr()
+}
+
+// ListItems queries the Gitea packages API for the configured org and builds a
+// types.RegistryContainer for the highest semver tag of each container package,
+// enriched with OCI labels (title, description, source repository, hidden flag).
+func ListItems(ctx context.Context) ([]types.RegistryContainer, error) {
+	// 1. Query Gitea REST API using the exact Org name
+	apiURL := fmt.Sprintf("%s/api/v1/packages/%s?type=container", BaseURL(), os.Getenv("GITEA_ORG"))
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create registry request: %w", err)
+	}
+
+	logger.Debug("Querying Gitea package API at %s", apiURL)
+	req.Header.Set("Authorization", "token "+os.Getenv("GITEA_RW_PACKAGE_TOKEN"))
+
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to reach Gitea package API: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, fmt.Errorf("registry API returned non-200 status code: %d", resp.StatusCode)
+	}
+
+	var rawPackages []types.RawPackage
+	if err := json.NewDecoder(resp.Body).Decode(&rawPackages); err != nil {
+		return nil, fmt.Errorf("failed to decode registry response: %w", err)
+	}
+
+	// 2. Group by Name and filter out "latest" & "sha256:"
+	groupedPackages := make(map[string][]types.RawPackage)
+	for _, pkg := range rawPackages {
+		if pkg.Version == "latest" || strings.HasPrefix(pkg.Version, "sha256:") {
+			continue
+		}
+		groupedPackages[pkg.Name] = append(groupedPackages[pkg.Name], pkg)
+	}
+
+	// 3. For each container name, find the package entry with the highest semver
+	var latestPackages []types.RawPackage
+	for _, pkgs := range groupedPackages {
+		if len(pkgs) == 0 {
+			continue
+		}
+
+		sort.Slice(pkgs, func(i, j int) bool {
+			nI := normalizeSemver(pkgs[i].Version)
+			nJ := normalizeSemver(pkgs[j].Version)
+			vI := "v" + nI
+			vJ := "v" + nJ
+
+			// Compare using semver package
+			if semver.IsValid(vI) && semver.IsValid(vJ) {
+				if c := semver.Compare(vI, vJ); c != 0 {
+					return c > 0
+				}
+			} else if nI != nJ {
+				return nI > nJ
+			}
+			// Tie-break on specificity (most complete wins)
+			if sI, sJ := semverSpecificity(pkgs[i].Version), semverSpecificity(pkgs[j].Version); sI != sJ {
+				return sI > sJ
+			}
+			// Tie-break on creation time (newest wins)
+			return pkgs[i].CreatedAt > pkgs[j].CreatedAt
+		})
+
+		// First item after descending sort is the highest version
+		latestPackages = append(latestPackages, pkgs[0])
+	}
+
+	ociAuth := remote.WithAuth(&authn.Basic{
+		Username: os.Getenv("GITEA_USER"),
+		Password: os.Getenv("GITEA_RW_PACKAGE_TOKEN"),
+	})
+
+	ociOrg := strings.ToLower(os.Getenv("GITEA_ORG"))
+	var registryItems []types.RegistryContainer
+
+	// 4. Fetch OCI metadata only for the highest version of each package
+	for _, pkg := range latestPackages {
+		pkgNameLower := strings.ToLower(pkg.Name)
+		imageRef := fmt.Sprintf("%s/%s/%s:%s", RegistryHost(), ociOrg, pkgNameLower, pkg.Version)
+
+		// The Gitea package API does not link org-level container packages to a
+		// repository, so the repository URL is resolved from the image's OCI
+		// labels below, or derived from the image reference as a last resort.
+		item := types.RegistryContainer{
+			Name:      pkg.Name,
+			Version:   normalizeSemver(pkg.Version),
+			Image:     imageRef,
+			CreatedAt: pkg.CreatedAt,
+		}
+
+		var refOpts []name.Option
+		if IsInsecure() {
+			refOpts = append(refOpts, name.Insecure)
+		}
+		ref, err := name.ParseReference(imageRef, refOpts...)
+		if err != nil {
+			logger.Error("Failed to parse image reference %s: %v", imageRef, err)
+			registryItems = append(registryItems, item)
+			continue
+		}
+
+		img, err := remote.Image(ref, ociAuth, remote.WithContext(ctx))
+		if err != nil {
+			logger.Error("Failed to fetch OCI metadata for %s: %v", imageRef, err)
+			registryItems = append(registryItems, item)
+			continue
+		}
+
+		// Fetch image digest (sha256:...)
+		if digest, err := img.Digest(); err == nil {
+			item.Digest = digest.String()
+		}
+
+		// 1. Extract standard OCI annotations from OCI Manifest
+		if manifest, err := img.Manifest(); err == nil && manifest != nil {
+			if title, ok := manifest.Annotations["org.opencontainers.image.title"]; ok {
+				item.Title = title
+			}
+			if vendor, ok := manifest.Annotations["org.opencontainers.image.vendor"]; ok {
+				item.Vendor = vendor
+			}
+			if author, ok := manifest.Annotations["org.opencontainers.image.authors"]; ok {
+				item.Author = author
+			}
+			if desc, ok := manifest.Annotations["org.opencontainers.image.description"]; ok {
+				item.Description = desc
+			}
+			if src, ok := manifest.Annotations["org.opencontainers.image.source"]; ok && src != "" {
+				item.Source = src
+			}
+			if rev, ok := manifest.Annotations["org.opencontainers.image.revision"]; ok {
+				item.Revision = rev
+			}
+			if lic, ok := manifest.Annotations["org.opencontainers.image.licenses"]; ok {
+				item.Licenses = lic
+			}
+			if url, ok := manifest.Annotations["org.opencontainers.image.url"]; ok {
+				item.URL = url
+			}
+			if doc, ok := manifest.Annotations["org.opencontainers.image.documentation"]; ok {
+				item.Documentation = doc
+			}
+			if created, ok := manifest.Annotations["org.opencontainers.image.created"]; ok && created != "" {
+				item.CreatedAt = created
+			}
+			if hidden, ok := manifest.Annotations["org.dcntrlized.hidden"]; ok && hidden == "true" {
+				item.Hidden = true
+			}
+		}
+
+		// 2. Fallback to Image Config Labels (where docker buildx puts them when targeting v2 registries)
+		if configFile, err := img.ConfigFile(); err == nil && configFile != nil {
+			labels := configFile.Config.Labels
+
+			if item.Title == "" {
+				item.Title = labels["org.opencontainers.image.title"]
+			}
+			if item.Vendor == "" {
+				item.Vendor = labels["org.opencontainers.image.vendor"]
+			}
+			if item.Author == "" {
+				item.Author = labels["org.opencontainers.image.authors"]
+			}
+			if item.Description == "" {
+				item.Description = labels["org.opencontainers.image.description"]
+			}
+			if src, ok := labels["org.opencontainers.image.source"]; ok && src != "" {
+				item.Source = src
+			}
+			if item.Revision == "" {
+				item.Revision = labels["org.opencontainers.image.revision"]
+			}
+			if item.Licenses == "" {
+				item.Licenses = labels["org.opencontainers.image.licenses"]
+			}
+			if item.URL == "" {
+				item.URL = labels["org.opencontainers.image.url"]
+			}
+			if item.Documentation == "" {
+				item.Documentation = labels["org.opencontainers.image.documentation"]
+			}
+			if item.CreatedAt == "" && labels["org.opencontainers.image.created"] != "" {
+				item.CreatedAt = labels["org.opencontainers.image.created"]
+			}
+			if !item.Hidden && labels["org.dcntrlized.hidden"] == "true" {
+				item.Hidden = true
+			}
+		}
+
+		// Fallback for infra images published before the hidden label existed:
+		// the shared database is always treated as hidden infrastructure.
+		if strings.EqualFold(item.Name, "dcntrlized-db") {
+			item.Hidden = true
+		}
+
+		// Resolve the source repository URL: OCI source label if present, else
+		// derived from the image reference (host/org/repo).
+		if item.Source == "" {
+			item.Source = RepoURLFromImageRef(imageRef)
+		}
+		// The compose file is fetched from the git repository, so fall back to
+		// the resolved source URL when the OCI url label is absent.
+		if item.URL == "" {
+			item.URL = item.Source
+		}
+
+		registryItems = append(registryItems, item)
+	}
+
+	return registryItems, nil
+}
+
+// ItemByName returns the registry item for the named service, or an error when
+// the registry does not carry a container package with that name.
+func ItemByName(ctx context.Context, name string) (*types.RegistryContainer, error) {
+	items, err := ListItems(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for i := range items {
+		if strings.EqualFold(items[i].Name, name) {
+			return &items[i], nil
+		}
+	}
+	return nil, fmt.Errorf("registry: service %q not found", name)
+}
+
+// normalizeSemver expands partial version tags to the full X.Y.Z form,
+// preserving an optional pre-release suffix (v3 -> 3.0.0, v3.1 -> 3.1.0,
+// v3.3.3-hotfix -> 3.3.3-hotfix).
+func normalizeSemver(v string) string {
+	tag := strings.TrimPrefix(v, "v")
+	prerelease := ""
+	if i := strings.IndexByte(tag, '-'); i >= 0 {
+		prerelease = tag[i:]
+		tag = tag[:i]
+	}
+	parts := strings.Split(tag, ".")
+	for len(parts) < 3 {
+		parts = append(parts, "0")
+	}
+	return strings.Join(parts, ".") + prerelease
+}
+
+// semverSpecificity counts the dot-separated segments of a version's
+// major.minor.patch part, so that among versions that normalize equal the most
+// complete one wins (3.4.0 over 3.4).
+func semverSpecificity(v string) int {
+	tag := strings.TrimPrefix(v, "v")
+	if i := strings.IndexByte(tag, '-'); i >= 0 {
+		tag = tag[:i]
+	}
+	return strings.Count(tag, ".")
+}
@@ -0,0 +1,52 @@
+// Copyright (C) 2026 Dcntrlized
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published
+// by the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package registry
+
+import (
+	"context"
+	"os"
+	"strings"
+	"testing"
+	"time"
+)
+
+// TestItemByNameLive resolves the infrastructure services against a live
+// registry. It is opt-in (MAESTRO_REGISTRY_TEST=1) so the default test run
+// stays offline; the dev VM and prod CI both set the Gitea env the function
+// reads. Infrastructure services must resolve so Maestro can auto-install them
+// on a fresh host (ensureInfraRunning).
+func TestItemByNameLive(t *testing.T) {
+	if os.Getenv("MAESTRO_REGISTRY_TEST") != "1" {
+		t.Skip("set MAESTRO_REGISTRY_TEST=1 with GITEA_* env to run against a live registry")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	for _, name := range []string{"dcntrlized-db", "dcntrlized-auth"} {
+		item, err := ItemByName(ctx, name)
+		if err != nil {
+			t.Fatalf("ItemByName(%q): %v", name, err)
+		}
+		if item.Image == "" {
+			t.Errorf("ItemByName(%q): empty image", name)
+		}
+		if !strings.HasPrefix(item.URL, "http") {
+			t.Errorf("ItemByName(%q): url %q not an http(s) compose source", name, item.URL)
+		}
+		t.Logf("resolved %s -> image=%s url=%s version=%s", name, item.Image, item.URL, item.Version)
+	}
+}