Skip to content

Commit 2fb4154

Browse files
johscheuerjohscheuer
authored
Rewrite data loader in golang (#2254)
1 parent 9654776 commit 2fb4154

File tree

5 files changed

+189
-67
lines changed

5 files changed

+189
-67
lines changed

‎sample-apps/data-loader/Dockerfile‎

Lines changed: 80 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,90 @@
1-
FROM docker.io/library/python:3.13-slim
1+
ARG FDB_VERSION=7.1.67
2+
ARG FDB_WEBSITE=https://github.com/apple/foundationdb/releases/download
23

4+
# Build the manager binary
5+
FROM golang:1.24.9-bookworm AS builder
6+
7+
ARG FDB_VERSION
8+
ARG FDB_WEBSITE
9+
ARG TARGETARCH
10+
ARG TAG="latest"
11+
12+
RUN set -eux && \
13+
if [ "$TARGETARCH" = "amd64" ]; then \
14+
FDB_ARCH=amd64; \
15+
elif [ "$TARGETARCH" = "arm64" ]; then \
16+
FDB_ARCH=aarch64; \
17+
if [ "${FDB_VERSION%.*}" = "7.1" ]; then \
18+
FDB_VERSION="7.3.71"; \
19+
fi; \
20+
else \
21+
echo "ERROR: unsupported architecture $TARGETARCH" 1>&2; \
22+
exit 1; \
23+
fi; \
24+
curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb" -o foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb && \
25+
curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb.sha256" -o foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb.sha256 && \
26+
sha256sum -c foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb.sha256 && \
27+
dpkg -i foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb && \
28+
rm foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb foundationdb-clients_${FDB_VERSION}-1_${FDB_ARCH}.deb.sha256
29+
30+
WORKDIR /workspace
31+
# Copy the Go Modules manifests
32+
COPY go.mod go.mod
33+
COPY go.sum go.sum
34+
# cache deps before building and copying source so that we don't need to re-download as much
35+
# and so that source changes don't invalidate our downloaded layer
36+
RUN go mod download -x
37+
38+
# Copy the go source
39+
COPY main.go main.go
40+
41+
# Build
42+
RUN CGO_ENABLED=1 GOOS=linux GOARCH=${TARGETARCH} GO111MODULE=on go build -o /workspace/bin/data-loader main.go
43+
44+
FROM rockylinux/rockylinux:9.6-minimal
45+
46+
ARG FDB_VERSION
47+
ARG FDB_WEBSITE
348
ARG TARGETARCH
449

5-
COPY app.py /usr/local/bin
50+
VOLUME /usr/lib/fdb
51+
52+
WORKDIR /
53+
54+
RUN set -eux && \
55+
if [ "$TARGETARCH" = "amd64" ]; then \
56+
FDB_ARCH=x86_64; \
57+
elif [ "$TARGETARCH" = "arm64" ]; then \
58+
FDB_ARCH=aarch64; \
59+
if [ "${FDB_VERSION%.*}" = "7.1" ]; then \
60+
FDB_VERSION="7.3.71"; \
61+
fi; \
62+
else \
63+
echo "ERROR: unsupported architecture $TARGETARCH" 1>&2; \
64+
exit 1; \
65+
fi; \
66+
if [ "${FDB_VERSION%.*}" = "7.1" ]; then \
67+
# FDB 7.1 published the client packages for el7, 7.3 and newer uses el9.
68+
FDB_OS=el7; \
69+
else \
70+
FDB_OS=el9; \
71+
fi; \
72+
curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm" -o foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm && \
73+
curl --fail -L "${FDB_WEBSITE}/${FDB_VERSION}/foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm.sha256" -o foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm.sha256 && \
74+
microdnf install -y glibc pkg-config && \
75+
microdnf clean all && \
76+
sha256sum -c foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm.sha256 && \
77+
rpm -i foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm --excludepath=/usr/bin --excludepath=/usr/lib/foundationdb/backup_agent && \
78+
rm foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm foundationdb-clients-${FDB_VERSION}-1.${FDB_OS}.${FDB_ARCH}.rpm.sha256
679

7-
RUN pip install foundationdb==7.1.67
880
RUN groupadd --gid 4059 fdb && \
9-
useradd --gid 4059 --uid 4059 --shell /usr/sbin/nologin fdb
81+
useradd --gid 4059 --uid 4059 --shell /usr/sbin/nologin fdb && \
82+
mkdir -p /var/log/fdb && \
83+
touch /var/log/fdb/.keep
1084

11-
RUN apt-get update && \
12-
apt-get install -y --no-install-recommends curl && \
13-
curl -L https://github.com/krallin/tini/releases/download/v0.19.0/tini-${TARGETARCH} -o tini-${TARGETARCH} && \
14-
echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini-amd64\n07952557df20bfd2a95f9bef198b445e006171969499a1d361bd9e6f8e5e0e81 tini-arm64" > tini-sha.txt && \
15-
sha256sum --quiet --ignore-missing -c tini-sha.txt && \
16-
chmod +x tini-${TARGETARCH} && \
17-
mv tini-${TARGETARCH} /usr/bin/tini && \
18-
rm -rf /tmp/*
85+
COPY --chown=fdb:fdb --from=builder /workspace/bin/data-loader /usr/local/bin/data-loader
1986

2087
# Set to the numeric UID of fdb user to satisfy PodSecurityPolices which enforce runAsNonRoot
2188
USER 4059
2289

23-
ENTRYPOINT [ "/usr/bin/tini", "-g", "--", "python", "/usr/local/bin/app.py" ]
90+
ENTRYPOINT ["/usr/local/bin/data-loader"]

‎sample-apps/data-loader/app.py‎

Lines changed: 0 additions & 54 deletions
This file was deleted.

‎sample-apps/data-loader/go.mod‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
module github.com/FoundationDB/fdb-data-loader
2+
3+
go 1.24.0
4+
5+
toolchain go1.24.4
6+
7+
require (
8+
// Binding version for 7.1.67
9+
github.com/apple/foundationdb/bindings/go v0.0.0-20250115161953-f1ab8147ed1c
10+
github.com/google/uuid v1.6.0
11+
)

‎sample-apps/data-loader/go.sum‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
github.com/apple/foundationdb/bindings/go v0.0.0-20250115161953-f1ab8147ed1c h1:Nnun3T50beIpO6YKDZInHuZMgnNtYJafSlQAvkXwOWc=
2+
github.com/apple/foundationdb/bindings/go v0.0.0-20250115161953-f1ab8147ed1c/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U=
3+
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
4+
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

‎sample-apps/data-loader/main.go‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"encoding/binary"
6+
"flag"
7+
"log"
8+
"math/rand/v2"
9+
"os"
10+
"os/signal"
11+
"syscall"
12+
"time"
13+
14+
"github.com/apple/foundationdb/bindings/go/src/fdb"
15+
"github.com/apple/foundationdb/bindings/go/src/fdb/tuple"
16+
"github.com/google/uuid"
17+
)
18+
19+
// initRandomGenerator setup the random generator to generate the values.
20+
func initRandomGenerator() *rand.ChaCha8 {
21+
bs := make([]byte, 8)
22+
binary.LittleEndian.PutUint64(bs, uint64(time.Now().UnixMilli()))
23+
24+
seed := [32]byte(make([]byte, 32))
25+
var idx int
26+
for idx < 32 {
27+
for _, b := range bs {
28+
seed[idx] = b
29+
idx++
30+
31+
if idx == 32 {
32+
break
33+
}
34+
}
35+
}
36+
37+
return rand.NewChaCha8(seed)
38+
}
39+
40+
func loadData(ctx context.Context, keys int, batchSize int, valueSize int, clusterFile string) {
41+
batchCount := keys / batchSize
42+
43+
log.Println("opening database with cluster file:", clusterFile)
44+
db, err := fdb.OpenDatabase(clusterFile)
45+
if err != nil {
46+
log.Fatalf("could not open database: %s", err)
47+
}
48+
49+
randomGen := initRandomGenerator()
50+
for i := 0; i < batchCount; i++ {
51+
select {
52+
case <-ctx.Done():
53+
// Handle graceful shutdown
54+
log.Printf("Stopping: %v\n", ctx.Err())
55+
return
56+
default:
57+
log.Println("Writing batch", i)
58+
59+
prefix := uuid.NewString()
60+
_, err = db.Transact(func(transaction fdb.Transaction) (interface{}, error) {
61+
for idx := 0; idx < batchSize; idx++ {
62+
token := make([]byte, valueSize)
63+
_, randErr := randomGen.Read(token)
64+
if randErr != nil {
65+
return nil, randErr
66+
}
67+
68+
transaction.Set(tuple.Tuple{prefix, idx}, token)
69+
}
70+
71+
return nil, nil
72+
})
73+
74+
if err != nil {
75+
log.Printf("could not write data: %s\n", err)
76+
}
77+
}
78+
}
79+
}
80+
81+
func main() {
82+
fdb.MustAPIVersion(710)
83+
84+
var keys, batchSize, valueSize int
85+
flag.IntVar(&keys, "keys", 100000, "Number of keys to generate")
86+
flag.IntVar(&batchSize, "batch-size", 10, "Number of updates per batch")
87+
flag.IntVar(&valueSize, "value-size", 1000, "Number of bytes to include in each value")
88+
flag.Parse()
89+
90+
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
91+
defer stop()
92+
93+
loadData(ctx, keys, batchSize, valueSize, os.Getenv("FDB_CLUSTER_FILE"))
94+
}

0 commit comments

Comments
 (0)