Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .custom-gcl.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
# with all the plugins listed below.
#
# See: https://golangci-lint.run/plugins/module-plugins/
version: v2.8.0
version: v2.15.0
plugins:
# logcheck validates structured logging calls and parameters (e.g., balanced key-value pairs)
- module: "sigs.k8s.io/logtools"
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.25"
go-version: "1.27.3"
cache-dependency-path: go.sum

# Checkout files are owned by the runner user. Without marking the workspace
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test-chart.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.25"
go-version: "1.27.3"
cache-dependency-path: go.sum

- name: Install CI dependencies
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.25"
go-version: "1.27.3"
cache-dependency-path: go.sum

- name: Install CI dependencies
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: "1.25"
go-version: "1.27.3"
cache-dependency-path: go.sum

# Required so that `go list`/`go test` can run `git` for VCS stamping
Expand Down
5 changes: 5 additions & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@ linters:
- dupl
- lll
path: internal/*
# goconst stopped ignoring tests (golangci-lint v2.8 -> v2.15); repeated
# literals in table tests are clearer as they are.
- linters:
- goconst
path: _test\.go
paths:
- third_party$
- builtin$
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# syntax=docker/dockerfile:1
# Build the manager binary
FROM golang:1.25 AS builder
FROM golang:1.27.3 AS builder
ARG TARGETOS
ARG TARGETARCH

Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ ENVTEST_K8S_VERSION ?= $(shell v='$(call gomodver,k8s.io/api)'; \
[ -n "$$v" ] || { echo "Set ENVTEST_K8S_VERSION manually (k8s.io/api replace has no tag)" >&2; exit 1; }; \
printf '%s\n' "$$v" | sed -E 's/^v?[0-9]+\.([0-9]+).*/1.\1/')

GOLANGCI_LINT_VERSION ?= v2.8.0
GOLANGCI_LINT_VERSION ?= v2.15.0
.PHONY: kustomize
kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
$(KUSTOMIZE): $(LOCALBIN)
Expand Down
55 changes: 41 additions & 14 deletions internal/controller/backup_job_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func testBackup() *hyperbytedbv1alpha1.HyperbytedbBackup {
}
}

func TestBuildBackupJobSnapshotsThenUploadsWithMc(t *testing.T) {
func TestBuildBackupJobSnapshotsThenUploadsWithAwsCli(t *testing.T) {
r := &HyperbytedbBackupReconciler{}
job := r.buildBackupJob(testBackup(), testBackupCluster(), "db-test-backup-abcd")
spec := job.Spec.Template.Spec
Expand All @@ -63,21 +63,22 @@ func TestBuildBackupJobSnapshotsThenUploadsWithMc(t *testing.T) {
}

if len(spec.Containers) != 1 {
t.Fatalf("containers = %d, want 1 (mc upload)", len(spec.Containers))
t.Fatalf("containers = %d, want 1 (S3 upload)", len(spec.Containers))
}
up := spec.Containers[0]
if up.Name != "upload" {
t.Errorf("container name = %q, want upload", up.Name)
}
if !strings.Contains(up.Image, "minio/mc") {
t.Errorf("upload image = %q, want minio/mc", up.Image)
// minio/mc is no longer pullable from Docker Hub.
if !strings.HasPrefix(up.Image, "amazon/aws-cli:") {
t.Errorf("upload image = %q, want amazon/aws-cli", up.Image)
}
upScript := strings.Join(up.Command, " ")
if !strings.Contains(upScript, "mc ") {
t.Errorf("upload command missing mc: %s", upScript)
if !strings.Contains(upScript, "aws $EP s3 sync /snapshot") {
t.Errorf("upload command missing aws s3 sync: %s", upScript)
}
if strings.Contains(upScript, "aws ") {
t.Errorf("upload must not call aws: %s", upScript)
if strings.Contains(upScript, "mc ") {
t.Errorf("upload must not call mc: %s", upScript)
}

names := map[string]bool{}
Expand All @@ -89,7 +90,7 @@ func TestBuildBackupJobSnapshotsThenUploadsWithMc(t *testing.T) {
}
}

func TestBuildRestoreJobDownloadsWithMcThenRestores(t *testing.T) {
func TestBuildRestoreJobDownloadsWithAwsCliThenRestores(t *testing.T) {
r := &HyperbytedbRestoreReconciler{}
restore := &hyperbytedbv1alpha1.HyperbytedbRestore{
ObjectMeta: metav1.ObjectMeta{Name: "db-test-restore-abcd", Namespace: "tenant-1"},
Expand All @@ -108,17 +109,17 @@ func TestBuildRestoreJobDownloadsWithMcThenRestores(t *testing.T) {
spec := job.Spec.Template.Spec

if len(spec.InitContainers) != 1 {
t.Fatalf("init containers = %d, want 1 (mc download)", len(spec.InitContainers))
t.Fatalf("init containers = %d, want 1 (S3 download)", len(spec.InitContainers))
}
dl := spec.InitContainers[0]
if dl.Name != "download" {
t.Errorf("init name = %q, want download", dl.Name)
}
if !strings.Contains(dl.Image, "minio/mc") {
t.Errorf("download image = %q, want minio/mc", dl.Image)
if !strings.HasPrefix(dl.Image, "amazon/aws-cli:") {
t.Errorf("download image = %q, want amazon/aws-cli", dl.Image)
}
if strings.Contains(strings.Join(dl.Command, " "), "aws ") {
t.Errorf("download must not call aws: %s", dl.Command)
if !strings.Contains(strings.Join(dl.Command, " "), "aws $EP s3 sync") {
t.Errorf("download command missing aws s3 sync: %s", dl.Command)
}

if len(spec.Containers) != 1 {
Expand All @@ -142,3 +143,29 @@ func TestBuildRestoreJobDownloadsWithMcThenRestores(t *testing.T) {
t.Fatalf("restore job name %q is %d bytes; Kubernetes job-name labels must be <=63", job.Name, n)
}
}

// A custom endpoint (MinIO) is addressed path-style through --endpoint-url;
// AWS itself by region, with neither.
func TestS3EnvAddressing(t *testing.T) {
env := func(spec hyperbytedbv1alpha1.S3BackupSpec) map[string]string {
out := map[string]string{}
for _, e := range s3Env(spec) {
out[e.Name] = e.Value
}
return out
}
minio := env(hyperbytedbv1alpha1.S3BackupSpec{Bucket: "b", Endpoint: "http://minio:9000"})
if minio["S3_PATH_STYLE"] != "1" || minio["S3_ENDPOINT"] != "http://minio:9000" {
t.Errorf("custom endpoint env = %v", minio)
}
aws := env(hyperbytedbv1alpha1.S3BackupSpec{Bucket: "b", Region: "eu-west-2"})
if _, ok := aws["S3_PATH_STYLE"]; ok {
t.Errorf("AWS must not use path-style: %v", aws)
}
if aws["AWS_DEFAULT_REGION"] != "eu-west-2" {
t.Errorf("region = %q", aws["AWS_DEFAULT_REGION"])
}
if got := env(hyperbytedbv1alpha1.S3BackupSpec{Bucket: "b"})["AWS_DEFAULT_REGION"]; got != "us-east-1" {
t.Errorf("default region = %q", got)
}
}
2 changes: 1 addition & 1 deletion internal/controller/hyperbytedbbackup_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ func (r *HyperbytedbBackupReconciler) buildBackupJob(backup *hyperbytedbv1alpha1
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyOnFailure,
InitContainers: []corev1.Container{snapshotBackupInitContainer(hyperbytedb.ResolveHyperbytedbImage(cluster))},
Containers: []corev1.Container{mcUploadContainer(s3, backup.Spec.RetentionDays)},
Containers: []corev1.Container{s3UploadContainer(s3, backup.Spec.RetentionDays)},
Volumes: []corev1.Volume{
dataVolume(cluster.Name, 0, true),
snapshotVolume(),
Expand Down
11 changes: 7 additions & 4 deletions internal/controller/hyperbytedbcluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ const clusterFinalizer = "hyperbytedb.hyperbyte.cloud/finalizer"
// nodes are within tolerance of each other on parquet file count.
const replicationStateHealthy = "Healthy"

// Condition set once the Raft cluster has formed.
const conditionClusterFormed = "ClusterFormed"

// HyperbytedbClusterReconciler reconciles a HyperbytedbCluster object.
type HyperbytedbClusterReconciler struct {
client.Client
Expand Down Expand Up @@ -192,7 +195,7 @@ func (r *HyperbytedbClusterReconciler) Reconcile(ctx context.Context, req ctrl.R
stsResult, err := r.reconcileStatefulSet(ctx, cluster, configHash)
if err != nil {
if apierrors.IsConflict(err) {
return ctrl.Result{Requeue: true}, nil
return ctrl.Result{RequeueAfter: time.Second}, nil
}
return r.setFailedStatus(ctx, cluster, "StatefulSetFailed", err)
}
Expand Down Expand Up @@ -1510,7 +1513,7 @@ func (r *HyperbytedbClusterReconciler) reconcileClusterMembership(
if leaderHost == "" {
log.V(1).Info("Could not locate raft leader from any pod; will retry next reconcile")
meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
Type: "ClusterFormed",
Type: conditionClusterFormed,
Status: metav1.ConditionFalse,
Reason: "NoLeader",
Message: "No raft leader reachable yet",
Expand Down Expand Up @@ -1586,15 +1589,15 @@ func (r *HyperbytedbClusterReconciler) reconcileClusterMembership(

if allMembers && len(knownIDs) >= int(replicas) {
meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
Type: "ClusterFormed",
Type: conditionClusterFormed,
Status: metav1.ConditionTrue,
Reason: "AllNodesMembers",
Message: fmt.Sprintf("All %d expected nodes are raft members", replicas),
LastTransitionTime: metav1.Now(),
})
} else {
meta.SetStatusCondition(&cluster.Status.Conditions, metav1.Condition{
Type: "ClusterFormed",
Type: conditionClusterFormed,
Status: metav1.ConditionFalse,
Reason: "MembershipIncomplete",
Message: fmt.Sprintf("%d/%d expected nodes are raft members",
Expand Down
2 changes: 1 addition & 1 deletion internal/controller/hyperbytedbrestore_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,7 @@ func (r *HyperbytedbRestoreReconciler) buildRestoreJob(
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyOnFailure,
InitContainers: []corev1.Container{mcDownloadInitContainer(*s3Source)},
InitContainers: []corev1.Container{s3DownloadInitContainer(*s3Source)},
Containers: []corev1.Container{restoreDataContainer(hyperbytedb.ResolveHyperbytedbImage(cluster), ordinal)},
Volumes: []corev1.Volume{
dataVolume(cluster.Name, ordinal, false),
Expand Down
Loading
Loading