diff --git a/internal/kustomize/kustomization.go b/internal/kustomize/kustomization.go new file mode 100644 index 00000000..de2ae2ef --- /dev/null +++ b/internal/kustomize/kustomization.go @@ -0,0 +1,491 @@ +/* +SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and component-operator-runtime contributors +SPDX-License-Identifier: Apache-2.0 +*/ + +package kustomize + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "reflect" + "strings" + "text/template" + + "github.com/Masterminds/sprig/v3" + "github.com/go-git/go-git/plumbing/format/gitignore" + "github.com/gobwas/glob" + "github.com/sap/go-generics/maps" + "github.com/sap/go-generics/slices" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/version" + "k8s.io/client-go/discovery" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/kustomize/api/konfig" + kustypes "sigs.k8s.io/kustomize/api/types" + kustfsys "sigs.k8s.io/kustomize/kyaml/filesys" + kyaml "sigs.k8s.io/yaml" + + "github.com/sap/component-operator-runtime/internal/fileutils" + "github.com/sap/component-operator-runtime/internal/templatex" + "github.com/sap/component-operator-runtime/pkg/component" + "github.com/sap/component-operator-runtime/pkg/manifests" +) + +// TODO: double-check symlink handling + +const ( + componentConfigFilename = ".component-config.yaml" + componentIgnoreFilename = ".component-ignore" +) + +type KustomizationOptions struct { + TemplateSuffix *string + // If defined, the given left delimiter will be used to parse go templates; otherwise, defaults to '{{' + LeftTemplateDelimiter *string + // If defined, the given right delimiter will be used to parse go templates; otherwise, defaults to '}}' + RightTemplateDelimiter *string + // If defined, paths to referenced files or directories outside kustomizationPath + IncludedFiles []string + // If defined, paths to referenced kustomizations + IncludedKustomizations []string + // If defined, used to decrypt files + Decryptor manifests.Decryptor +} + +type RenderContext struct { + LocalClient client.Client + Client client.Client + DiscoveryClient discovery.DiscoveryInterface + Component component.Component + ComponentDigest string + Namespace string + Name string + Parameters map[string]any +} + +type Kustomization struct { + path string + files map[string][]byte + nonTemplates map[string][]byte + templates map[string]*template.Template + kustomizations []*Kustomization +} + +// TODO: add a way to pass custom template functions + +func ParseKustomization(fsys fs.FS, kustomizationPath string, options KustomizationOptions) (*Kustomization, error) { + kustomization, err := parseKustomization(fsys, kustomizationPath, options, nil) + if err != nil { + return nil, err + } + + return kustomization, nil +} + +func parseKustomization(fsys fs.FS, kustomizationPath string, options KustomizationOptions, visitedKustomizationPaths []string) (*Kustomization, error) { + if options.TemplateSuffix == nil { + options.TemplateSuffix = ref("") + } + if options.LeftTemplateDelimiter == nil { + options.LeftTemplateDelimiter = ref("") + } + if options.RightTemplateDelimiter == nil { + options.RightTemplateDelimiter = ref("") + } + + if fsys == nil { + fsys = os.DirFS("/") + absoluteKustomizationPath, err := filepath.Abs(kustomizationPath) + if err != nil { + return nil, err + } + kustomizationPath = absoluteKustomizationPath[1:] + } else if filepath.IsAbs(kustomizationPath) { + kustomizationPath = kustomizationPath[1:] + } + kustomizationPath = filepath.Clean(kustomizationPath) + if slices.Any(visitedKustomizationPaths, func(path string) bool { + return isSubdirectory(kustomizationPath, path) + }) { + return nil, fmt.Errorf("path %s part of another referenced kustomization", kustomizationPath) + } + visitedKustomizationPaths = append(visitedKustomizationPaths, kustomizationPath) + + if info, err := fs.Stat(fsys, kustomizationPath); err != nil { + return nil, err + } else if !info.IsDir() { + return nil, fmt.Errorf("path %s is not a directory", kustomizationPath) + } + + k := Kustomization{ + path: kustomizationPath, + files: make(map[string][]byte), + nonTemplates: make(map[string][]byte), + templates: make(map[string]*template.Template), + } + + if err := readOptions(fsys, filepath.Clean(filepath.Join(kustomizationPath, componentConfigFilename)), &options); err != nil { + return nil, err + } + + ignore, err := readIgnore(fsys, filepath.Clean(filepath.Join(kustomizationPath, componentIgnoreFilename))) + if err != nil { + return nil, err + } + + var t *template.Template + files, err := fileutils.Find(fsys, kustomizationPath, "*", fileutils.FileTypeRegular, 0) + if err != nil { + return nil, err + } + for _, file := range files { + raw, err := fs.ReadFile(fsys, file) + if err != nil { + return nil, err + } + if options.Decryptor != nil { + raw, err = options.Decryptor.Decrypt(raw, file) + if err != nil { + return nil, err + } + } + name, err := filepath.Rel(kustomizationPath, file) + if err != nil { + // TODO: is it ok to panic here in case of error ? + panic("this cannot happen") + } + k.files[name] = raw + if filepath.Base(name) == componentConfigFilename || filepath.Base(name) == componentIgnoreFilename { + continue + } + if ignore != nil && ignore.Match(filepath.SplitList(name), false) { + continue + } + if strings.HasSuffix(name, *options.TemplateSuffix) { + if t == nil { + t = template.New(name) + t.Delims(*options.LeftTemplateDelimiter, *options.RightTemplateDelimiter) + t.Option("missingkey=zero"). + Funcs(sprig.TxtFuncMap()). + Funcs(templatex.FuncMap()). + Funcs(templatex.FuncMapForTemplate(nil)). + Funcs(templatex.FuncMapForLocalClient(nil)). + Funcs(templatex.FuncMapForClient(nil)). + Funcs(funcMapForContext(nil, nil, nil, nil, "", "", "")) + } else { + t = t.New(name) + } + if _, err := t.Parse(string(raw)); err != nil { + return nil, err + } + k.templates[strings.TrimSuffix(name, *options.TemplateSuffix)] = t + } else { + k.nonTemplates[name] = raw + } + } + + // TODO: check that k.nonTemplates and k.templates are disjoint + + for _, path := range options.IncludedFiles { + if filepath.IsAbs(path) { + return nil, fmt.Errorf("include path (%s) must be absolute", path) + } + absolutePath := filepath.Clean(filepath.Join(kustomizationPath, path)) + if isSubdirectory(absolutePath, kustomizationPath) { + return nil, fmt.Errorf("include path (%s) must not be in the kustomization path (%s)", path, kustomizationPath) + } + if info, err := fs.Stat(fsys, absolutePath); err != nil { + return nil, err + } else if info.IsDir() { + files, err := fileutils.Find(fsys, absolutePath, "*", fileutils.FileTypeRegular, 0) + if err != nil { + return nil, err + } + for _, file := range files { + raw, err := fs.ReadFile(fsys, file) + if err != nil { + return nil, err + } + if options.Decryptor != nil { + raw, err = options.Decryptor.Decrypt(raw, file) + if err != nil { + return nil, err + } + } + k.files[path] = raw + } + } else { + raw, err := fs.ReadFile(fsys, absolutePath) + if err != nil { + return nil, err + } + if options.Decryptor != nil { + raw, err = options.Decryptor.Decrypt(raw, absolutePath) + if err != nil { + return nil, err + } + } + k.files[path] = raw + } + } + + for _, path := range options.IncludedKustomizations { + if filepath.IsAbs(path) { + return nil, fmt.Errorf("include path (%s) must be absolute", path) + } + absolutePath := filepath.Clean(filepath.Join(kustomizationPath, path)) + if isSubdirectory(absolutePath, kustomizationPath) { + // this is actually redundant; the same is checked through via visitedKustomizationPaths when calling parseKustomization(); + // but we keep it to maintain symmetry with the IncludedFiles handling, and because of the better error message + return nil, fmt.Errorf("include path (%s) must not be in the kustomization path (%s)", path, kustomizationPath) + } + kustomization, err := parseKustomization(fsys, absolutePath, KustomizationOptions{}, visitedKustomizationPaths) + if err != nil { + return nil, err + } + k.kustomizations = append(k.kustomizations, kustomization) + } + + return &k, nil +} + +func (k *Kustomization) Path() string { + return k.path +} + +func (k *Kustomization) Render(context RenderContext, fsys kustfsys.FileSystem) error { + serverVersion, err := context.DiscoveryClient.ServerVersion() + if err != nil { + return err + } + _, serverGroupsWithResources, err := context.DiscoveryClient.ServerGroupsAndResources() + if err != nil { + return err + } + serverGroupsWithResources = normalizeServerGroupsWithResources(serverGroupsWithResources) + + data := context.Parameters + + for n, f := range k.nonTemplates { + if err := fsys.WriteFile(filepath.Join(k.path, n), f); err != nil { + return err + } + } + + var t0 *template.Template + for n, t := range k.templates { + if t0 == nil { + t0, err = t.Clone() + if err != nil { + return err + } + t0.Option("missingkey=zero"). + Funcs(templatex.FuncMapForTemplate(t0)). + Funcs(templatex.FuncMapForLocalClient(context.LocalClient)). + Funcs(templatex.FuncMapForClient(context.Client)). + Funcs(funcMapForContext(k.files, serverVersion, serverGroupsWithResources, context.Component, context.ComponentDigest, context.Namespace, context.Name)) + } + var buf bytes.Buffer + // TODO: templates (accidentally or intentionally) could modify data, or even some of the objects supplied through builtin functions; + // such as serverVersion or component; this should be hardened, e.g. by deep-copying things upfront, or serializing them; see the comment in + // funcMapForContext() + if err := t0.ExecuteTemplate(&buf, t.Name(), data); err != nil { + return err + } + if err := fsys.WriteFile(filepath.Join(k.path, n), templatex.AdjustTemplateOutput(buf.Bytes())); err != nil { + return err + } + } + + haveKustomization := false + for _, kustomizationName := range konfig.RecognizedKustomizationFileNames() { + if fsys.Exists(filepath.Join(k.path, kustomizationName)) { + haveKustomization = true + break + } + } + if !haveKustomization { + kustomization, err := generateKustomization(fsys, k.path) + if err != nil { + return err + } + if err := fsys.WriteFile(filepath.Join(k.path, konfig.DefaultKustomizationFileName()), kustomization); err != nil { + return err + } + } + + for _, kustomization := range k.kustomizations { + if err := kustomization.Render(context, fsys); err != nil { + return err + } + } + + return nil +} + +func funcMapForContext(files map[string][]byte, serverInfo *version.Info, serverGroupsWithResources []*metav1.APIResourceList, component component.Component, componentDigest string, namespace string, name string) template.FuncMap { + return template.FuncMap{ + // TODO: maybe it would it be better to convert component to unstructured; + // then calling methods would no longer be possible, and attributes would be in lowercase + "listFiles": makeFuncListFiles(files), + "existsFile": makeFuncExistsFile(files), + "readFile": makeFuncReadFile(files), + "component": makeFuncData(component), + "componentDigest": func() string { return componentDigest }, + "namespace": func() string { return namespace }, + "name": func() string { return name }, + "kubernetesVersion": func() *version.Info { return serverInfo }, + "apiResources": func() []*metav1.APIResourceList { return serverGroupsWithResources }, + } +} + +func makeFuncListFiles(files map[string][]byte) func(pattern string) ([]string, error) { + return func(pattern string) ([]string, error) { + g, err := glob.Compile(pattern, '/') + if err != nil { + return nil, err + } + return slices.Sort(slices.Select(maps.Keys(files), func(path string) bool { return g.Match(path) })), nil + } +} + +func makeFuncExistsFile(files map[string][]byte) func(path string) bool { + return func(path string) bool { + _, ok := files[path] + return ok + } +} + +func makeFuncReadFile(files map[string][]byte) func(path string) ([]byte, error) { + return func(path string) ([]byte, error) { + data, ok := files[path] + if !ok { + return nil, fs.ErrNotExist + } + return data, nil + } +} + +func makeFuncData(data any) any { + if data == nil { + return func() any { return nil } + } + ival := reflect.ValueOf(data) + ityp := ival.Type() + ftyp := reflect.FuncOf(nil, []reflect.Type{ityp}, false) + fval := reflect.MakeFunc(ftyp, func(args []reflect.Value) []reflect.Value { return []reflect.Value{ival} }) + return fval.Interface() +} + +// TODO: this could be simplified; the files which will be considered as resources for the generated kustomization.yaml are +// exactly those keys of the k.templates and k.nonTemplates that do not start with '.' and do not end with '.yaml' or '.yml'; +// so the fsys.Walk below is basically unnecessary +func generateKustomization(fsys kustfsys.FileSystem, kustomizationPath string) ([]byte, error) { + var resources []string + + f := func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + // TODO: IsDir() is false if it is a symlink; is that wanted to be this way? + if !info.IsDir() && !strings.HasPrefix(filepath.Base(path), ".") && (strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")) { + resources = append(resources, path) + } + return nil + } + + // TODO: does this work correctly with symlinks? + if err := fsys.Walk(kustomizationPath, f); err != nil { + return nil, err + } + + kustomization := kustypes.Kustomization{ + TypeMeta: kustypes.TypeMeta{ + APIVersion: kustypes.KustomizationVersion, + Kind: kustypes.KustomizationKind, + }, + Resources: resources, + } + + if len(resources) == 0 { + // if there are no resources, set a dummy namespace to avoid "kustomization.yaml is empty" build error + kustomization.Namespace = "_dummy" + } + + rawKustomization, err := kyaml.Marshal(kustomization) + if err != nil { + return nil, err + } + + return rawKustomization, nil +} + +func readOptions(fsys fs.FS, path string, options *KustomizationOptions) error { + rawOptions, err := fs.ReadFile(fsys, path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + + if err := kyaml.Unmarshal(rawOptions, options); err != nil { + return err + } + + return nil +} + +func readIgnore(fsys fs.FS, path string) (gitignore.Matcher, error) { + var patterns []gitignore.Pattern + + ignoreFile, err := fsys.Open(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, err + } + defer ignoreFile.Close() + + domain := filepath.SplitList(path) + domain = domain[0 : len(domain)-1] + scanner := bufio.NewScanner(ignoreFile) + for scanner.Scan() { + s := scanner.Text() + if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 { + patterns = append(patterns, gitignore.ParsePattern(s, domain)) + } + } + + return gitignore.NewMatcher(patterns), nil +} + +func normalizeServerGroupsWithResources(serverGroupsWithResources []*metav1.APIResourceList) []*metav1.APIResourceList { + serverGroupsWithResources = slices.SortBy(serverGroupsWithResources, func(x, y *metav1.APIResourceList) bool { return x.GroupVersion > y.GroupVersion }) + for _, serverGroupWithResources := range serverGroupsWithResources { + serverGroupWithResources.APIResources = normalizeApiResources(serverGroupWithResources.APIResources) + } + return serverGroupsWithResources +} + +func normalizeApiResources(apiResources []metav1.APIResource) []metav1.APIResource { + apiResources = slices.SortBy(apiResources, func(x, y metav1.APIResource) bool { return x.Name > y.Name }) + for i := 0; i < len(apiResources); i++ { + apiResources[i].Verbs = slices.Sort(apiResources[i].Verbs) + apiResources[i].ShortNames = slices.Sort(apiResources[i].ShortNames) + apiResources[i].Categories = slices.Sort(apiResources[i].Categories) + } + return apiResources +} + +func isSubdirectory(subdir string, dir string) bool { + return subdir == dir || strings.HasPrefix(subdir, dir+string(filepath.Separator)) +} diff --git a/pkg/manifests/kustomize/util.go b/internal/kustomize/util.go similarity index 100% rename from pkg/manifests/kustomize/util.go rename to internal/kustomize/util.go diff --git a/pkg/manifests/kustomize/generator.go b/pkg/manifests/kustomize/generator.go index 6c721380..a3ae4561 100644 --- a/pkg/manifests/kustomize/generator.go +++ b/pkg/manifests/kustomize/generator.go @@ -6,68 +6,41 @@ SPDX-License-Identifier: Apache-2.0 package kustomize import ( - "bufio" "bytes" "context" - "errors" "io" "io/fs" - "os" - "path/filepath" - "reflect" - "strings" - "text/template" - "github.com/Masterminds/sprig/v3" - "github.com/go-git/go-git/plumbing/format/gitignore" - "github.com/gobwas/glob" - "github.com/sap/go-generics/maps" - "github.com/sap/go-generics/slices" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" utilyaml "k8s.io/apimachinery/pkg/util/yaml" - "k8s.io/apimachinery/pkg/version" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/kustomize/api/konfig" "sigs.k8s.io/kustomize/api/krusty" kustypes "sigs.k8s.io/kustomize/api/types" kustfsys "sigs.k8s.io/kustomize/kyaml/filesys" - kyaml "sigs.k8s.io/yaml" - "github.com/sap/component-operator-runtime/internal/fileutils" - "github.com/sap/component-operator-runtime/internal/templatex" + "github.com/sap/component-operator-runtime/internal/kustomize" "github.com/sap/component-operator-runtime/pkg/component" "github.com/sap/component-operator-runtime/pkg/manifests" "github.com/sap/component-operator-runtime/pkg/types" ) -// TODO: carve out logic into an internal Kustomization type (similar to the helm Chart case) -// TODO: double-check symlink handling - -const ( - componentConfigFilename = ".component-config.yaml" - componentIgnoreFilename = ".component-ignore" -) - // KustomizeGeneratorOptions allows to tweak the behavior of the kustomize generator. type KustomizeGeneratorOptions struct { - // If defined, only files with that suffix will be subject to templating. TemplateSuffix *string // If defined, the given left delimiter will be used to parse go templates; otherwise, defaults to '{{' LeftTemplateDelimiter *string // If defined, the given right delimiter will be used to parse go templates; otherwise, defaults to '}}' RightTemplateDelimiter *string + // If defined, used to decrypt files + Decryptor manifests.Decryptor } // KustomizeGenerator is a Generator implementation that basically renders a given Kustomization. // Note: KustomizeGenerator's Generate() method expects local client, client and component to be set in the passed context; // see: Context.WithLocalClient(), Context.WithClient() and Context.WithComponent() in package pkg/component. type KustomizeGenerator struct { - kustomizer *krusty.Kustomizer - files map[string][]byte - nonTemplates map[string][]byte - templates map[string]*template.Template + kustomization *kustomize.Kustomization + kustomizer *krusty.Kustomizer } var _ manifests.Generator = &KustomizeGenerator{} @@ -81,102 +54,26 @@ var _ manifests.Generator = &KustomizeGenerator{} // into a relative path by stripping the leading slash. If fsys is specified as a real filesystem, it is recommended to use os.Root.FS() instead of os.DirFS(), in order // to fence symbolic links. An empty kustomizationPath will be treated like ".". func NewKustomizeGenerator(fsys fs.FS, kustomizationPath string, _ client.Client, options KustomizeGeneratorOptions) (*KustomizeGenerator, error) { - if options.TemplateSuffix == nil { - options.TemplateSuffix = ref("") - } - if options.LeftTemplateDelimiter == nil { - options.LeftTemplateDelimiter = ref("") - } - if options.RightTemplateDelimiter == nil { - options.RightTemplateDelimiter = ref("") - } - - g := KustomizeGenerator{ - files: make(map[string][]byte), - nonTemplates: make(map[string][]byte), - templates: make(map[string]*template.Template), - } - - if fsys == nil { - fsys = os.DirFS("/") - absoluteKustomizationPath, err := filepath.Abs(kustomizationPath) - if err != nil { - return nil, err - } - kustomizationPath = absoluteKustomizationPath[1:] - } else if filepath.IsAbs(kustomizationPath) { - kustomizationPath = kustomizationPath[1:] + kustomization, err := kustomize.ParseKustomization(fsys, kustomizationPath, kustomize.KustomizationOptions{ + TemplateSuffix: options.TemplateSuffix, + LeftTemplateDelimiter: options.LeftTemplateDelimiter, + RightTemplateDelimiter: options.RightTemplateDelimiter, + Decryptor: options.Decryptor, + }) + if err != nil { + return nil, err } - kustomizationPath = filepath.Clean(kustomizationPath) kustomizerOptions := &krusty.Options{ LoadRestrictions: kustypes.LoadRestrictionsNone, PluginConfig: kustypes.DisabledPluginConfig(), } - g.kustomizer = krusty.MakeKustomizer(kustomizerOptions) - - if err := readOptions(fsys, filepath.Clean(kustomizationPath+"/"+componentConfigFilename), &options); err != nil { - return nil, err - } - - ignore, err := readIgnore(fsys, filepath.Clean(kustomizationPath+"/"+componentIgnoreFilename)) - if err != nil { - return nil, err - } + kustomizer := krusty.MakeKustomizer(kustomizerOptions) - var t *template.Template - // TODO: we should consider the whole of fsys, not only the subtree rooted at kustomizationPath; - // this would allow people to reference resources or patches or components located in parent directories - // (which is probably a common usecase); however it has to be clarified how to handle template scopes; - // for example it might be desired that subtrees with a kustomization.yaml file are processed in an own - // template context - files, err := fileutils.Find(fsys, kustomizationPath, "*", fileutils.FileTypeRegular, 0) - if err != nil { - return nil, err - } - for _, file := range files { - raw, err := fs.ReadFile(fsys, file) - if err != nil { - return nil, err - } - name, err := filepath.Rel(kustomizationPath, file) - if err != nil { - // TODO: is it ok to panic here in case of error ? - panic("this cannot happen") - } - g.files[name] = raw - if filepath.Base(name) == componentConfigFilename || filepath.Base(name) == componentIgnoreFilename { - continue - } - if ignore != nil && ignore.Match(filepath.SplitList(name), false) { - continue - } - if strings.HasSuffix(name, *options.TemplateSuffix) { - if t == nil { - t = template.New(name) - t.Delims(*options.LeftTemplateDelimiter, *options.RightTemplateDelimiter) - t.Option("missingkey=zero"). - Funcs(sprig.TxtFuncMap()). - Funcs(templatex.FuncMap()). - Funcs(templatex.FuncMapForTemplate(nil)). - Funcs(templatex.FuncMapForLocalClient(nil)). - Funcs(templatex.FuncMapForClient(nil)). - Funcs(funcMapForGenerateContext(nil, nil, nil, nil, "", "", "")) - } else { - t = t.New(name) - } - if _, err := t.Parse(string(raw)); err != nil { - return nil, err - } - g.templates[strings.TrimSuffix(name, *options.TemplateSuffix)] = t - } else { - g.nonTemplates[name] = raw - } - } - - // TODO: check that g.nonTemplates and g.templates are disjoint - - return &g, nil + return &KustomizeGenerator{ + kustomization: kustomization, + kustomizer: kustomizer, + }, nil } // Create a new KustomizeGenerator as TransformableGenerator. @@ -209,6 +106,7 @@ func NewKustomizeGeneratorWithObjectTransformer(fsys fs.FS, kustomizationPath st // Generate resource descriptors. func (g *KustomizeGenerator) Generate(ctx context.Context, namespace string, name string, parameters types.Unstructurable) ([]client.Object, error) { var objects []client.Object + fsys := kustfsys.MakeFsInMemory() localClient, err := component.LocalClientFromContext(ctx) if err != nil { @@ -226,68 +124,21 @@ func (g *KustomizeGenerator) Generate(ctx context.Context, namespace string, nam if err != nil { return nil, err } - serverVersion, err := clnt.DiscoveryClient().ServerVersion() - if err != nil { - return nil, err - } - _, serverGroupsWithResources, err := clnt.DiscoveryClient().ServerGroupsAndResources() - if err != nil { - return nil, err - } - serverGroupsWithResources = normalizeServerGroupsWithResources(serverGroupsWithResources) - data := parameters.ToUnstructured() - fsys := kustfsys.MakeFsInMemory() - - for n, f := range g.nonTemplates { - if err := fsys.WriteFile(n, f); err != nil { - return nil, err - } - } - - var t0 *template.Template - for n, t := range g.templates { - if t0 == nil { - t0, err = t.Clone() - if err != nil { - return nil, err - } - t0.Option("missingkey=zero"). - Funcs(templatex.FuncMapForTemplate(t0)). - Funcs(templatex.FuncMapForLocalClient(localClient)). - Funcs(templatex.FuncMapForClient(clnt)). - Funcs(funcMapForGenerateContext(g.files, serverVersion, serverGroupsWithResources, component, componentDigest, namespace, name)) - } - var buf bytes.Buffer - // TODO: templates (accidentally or intentionally) could modify data, or even some of the objects supplied through builtin functions; - // such as serverVersion or component; this should be hardened, e.g. by deep-copying things upfront, or serializing them; see the comment in - // funcMapForGenerateContext() - if err := t0.ExecuteTemplate(&buf, t.Name(), data); err != nil { - return nil, err - } - if err := fsys.WriteFile(n, templatex.AdjustTemplateOutput(buf.Bytes())); err != nil { - return nil, err - } - } - - haveKustomization := false - for _, kustomizationName := range konfig.RecognizedKustomizationFileNames() { - if fsys.Exists(kustomizationName) { - haveKustomization = true - break - } - } - if !haveKustomization { - kustomization, err := generateKustomization(fsys) - if err != nil { - return nil, err - } - if err := fsys.WriteFile(konfig.DefaultKustomizationFileName(), kustomization); err != nil { - return nil, err - } + if err := g.kustomization.Render(kustomize.RenderContext{ + LocalClient: localClient, + Client: clnt, + DiscoveryClient: clnt.DiscoveryClient(), + Component: component, + ComponentDigest: componentDigest, + Namespace: namespace, + Name: name, + Parameters: parameters.ToUnstructured(), + }, fsys); err != nil { + return nil, err } - resmap, err := g.kustomizer.Run(fsys, "/") + resmap, err := g.kustomizer.Run(fsys, g.kustomization.Path()) if err != nil { return nil, err } @@ -314,159 +165,3 @@ func (g *KustomizeGenerator) Generate(ctx context.Context, namespace string, nam return objects, nil } - -func funcMapForGenerateContext(files map[string][]byte, serverInfo *version.Info, serverGroupsWithResources []*metav1.APIResourceList, component component.Component, componentDigest string, namespace string, name string) template.FuncMap { - return template.FuncMap{ - // TODO: maybe it would it be better to convert component to unstructured; - // then calling methods would no longer be possible, and attributes would be in lowercase - "listFiles": makeFuncListFiles(files), - "existsFile": makeFuncExistsFile(files), - "readFile": makeFuncReadFile(files), - "component": makeFuncData(component), - "componentDigest": func() string { return componentDigest }, - "namespace": func() string { return namespace }, - "name": func() string { return name }, - "kubernetesVersion": func() *version.Info { return serverInfo }, - "apiResources": func() []*metav1.APIResourceList { return serverGroupsWithResources }, - } -} - -func makeFuncListFiles(files map[string][]byte) func(pattern string) ([]string, error) { - return func(pattern string) ([]string, error) { - g, err := glob.Compile(pattern, '/') - if err != nil { - return nil, err - } - return slices.Sort(slices.Select(maps.Keys(files), func(path string) bool { return g.Match(path) })), nil - } -} - -func makeFuncExistsFile(files map[string][]byte) func(path string) bool { - return func(path string) bool { - _, ok := files[path] - return ok - } -} - -func makeFuncReadFile(files map[string][]byte) func(path string) ([]byte, error) { - return func(path string) ([]byte, error) { - data, ok := files[path] - if !ok { - return nil, fs.ErrNotExist - } - return data, nil - } -} - -func makeFuncData(data any) any { - if data == nil { - return func() any { return nil } - } - ival := reflect.ValueOf(data) - ityp := ival.Type() - ftyp := reflect.FuncOf(nil, []reflect.Type{ityp}, false) - fval := reflect.MakeFunc(ftyp, func(args []reflect.Value) []reflect.Value { return []reflect.Value{ival} }) - return fval.Interface() -} - -// TODO: this could be simplified; the files which will be considered as resources for the generation kustomization.yaml are -// exactly those keys of the g.templates and g.nonTemplates that do not start with '.' and do not end with '.yaml' or '.yml'; -// so the fsys.Walk below is basically unnecessary -func generateKustomization(fsys kustfsys.FileSystem) ([]byte, error) { - var resources []string - - f := func(path string, info fs.FileInfo, err error) error { - if err != nil { - return err - } - // TODO: IsDir() is false if it is a symlink; is that wanted to be this way? - if !info.IsDir() && !strings.HasPrefix(filepath.Base(path), ".") && (strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")) { - resources = append(resources, path) - } - return nil - } - - // TODO: does this work correctly with symlinks? - if err := fsys.Walk(".", f); err != nil { - return nil, err - } - - kustomization := kustypes.Kustomization{ - TypeMeta: kustypes.TypeMeta{ - APIVersion: kustypes.KustomizationVersion, - Kind: kustypes.KustomizationKind, - }, - Resources: resources, - } - - if len(resources) == 0 { - // if there are no resources, set a dummy namespace to avoid "kustomization.yaml is empty" build error - kustomization.Namespace = "_dummy" - } - - rawKustomization, err := kyaml.Marshal(kustomization) - if err != nil { - return nil, err - } - - return rawKustomization, nil -} - -func readOptions(fsys fs.FS, path string, options *KustomizeGeneratorOptions) error { - rawOptions, err := fs.ReadFile(fsys, path) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return nil - } - return err - } - - if err := kyaml.Unmarshal(rawOptions, options); err != nil { - return err - } - - return nil -} - -func readIgnore(fsys fs.FS, path string) (gitignore.Matcher, error) { - var patterns []gitignore.Pattern - - ignoreFile, err := fsys.Open(path) - if err != nil { - if errors.Is(err, fs.ErrNotExist) { - return nil, nil - } - return nil, err - } - defer ignoreFile.Close() - - domain := filepath.SplitList(path) - domain = domain[0 : len(domain)-1] - scanner := bufio.NewScanner(ignoreFile) - for scanner.Scan() { - s := scanner.Text() - if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 { - patterns = append(patterns, gitignore.ParsePattern(s, domain)) - } - } - - return gitignore.NewMatcher(patterns), nil -} - -func normalizeServerGroupsWithResources(serverGroupsWithResources []*metav1.APIResourceList) []*metav1.APIResourceList { - serverGroupsWithResources = slices.SortBy(serverGroupsWithResources, func(x, y *metav1.APIResourceList) bool { return x.GroupVersion > y.GroupVersion }) - for _, serverGroupWithResources := range serverGroupsWithResources { - serverGroupWithResources.APIResources = normalizeApiResources(serverGroupWithResources.APIResources) - } - return serverGroupsWithResources -} - -func normalizeApiResources(apiResources []metav1.APIResource) []metav1.APIResource { - apiResources = slices.SortBy(apiResources, func(x, y metav1.APIResource) bool { return x.Name > y.Name }) - for i := 0; i < len(apiResources); i++ { - apiResources[i].Verbs = slices.Sort(apiResources[i].Verbs) - apiResources[i].ShortNames = slices.Sort(apiResources[i].ShortNames) - apiResources[i].Categories = slices.Sort(apiResources[i].Categories) - } - return apiResources -} diff --git a/pkg/manifests/types.go b/pkg/manifests/types.go index 4dc94e97..ac0c2be4 100644 --- a/pkg/manifests/types.go +++ b/pkg/manifests/types.go @@ -41,3 +41,9 @@ type ParameterTransformer interface { type ObjectTransformer interface { TransformObjects(namespace string, name string, objects []client.Object) ([]client.Object, error) } + +// Decryptor interface. +// Allows to decrypt content of referenced manifest sources. +type Decryptor interface { + Decrypt(input []byte, path string) ([]byte, error) +} diff --git a/website/content/en/docs/generators/kustomize.md b/website/content/en/docs/generators/kustomize.md index 63dda488..38fcc62c 100644 --- a/website/content/en/docs/generators/kustomize.md +++ b/website/content/en/docs/generators/kustomize.md @@ -87,20 +87,41 @@ Here: package kustomize type KustomizeGeneratorOptions struct { - // If defined, only files with that suffix will be subject to templating. + TemplateSuffix *string + // If defined, the given left delimiter will be used to parse go templates; + // otherwise, defaults to '{{' + LeftTemplateDelimiter *string + // If defined, the given right delimiter will be used to parse go templates; + // otherwise, defaults to '}}' + RightTemplateDelimiter *string + // If defined, used to decrypt files + Decryptor manifests.Decryptor + } + ``` + + The generator options can be overridden on source level by creating a file `.component-config.yaml` in the specified `kustomizationPath`; the file can contain JSON or YAML, compatible with the `KustomizationsOptions` struct: + + ```go + package kustomize + + type KustomizationOptions struct { TemplateSuffix *string - // If defined, the given left delimiter will be used to parse go templates; - // otherwise, defaults to '{{' + // If defined, the given left delimiter will be used to parse go templates; otherwise, defaults to '{{' LeftTemplateDelimiter *string - // If defined, the given right delimiter will be used to parse go templates; - // otherwise, defaults to '}}' + // If defined, the given right delimiter will be used to parse go templates; otherwise, defaults to '}}' RightTemplateDelimiter *string + // If defined, paths to referenced files or directories outside kustomizationPath + IncludedFiles []string + // If defined, paths to referenced kustomizations + IncludedKustomizations []string + // If defined, used to decrypt files + Decryptor manifests.Decryptor } ``` - The generator options can be overridden on source level by creating a file `.component-config.yaml` in the specified `kustomizationPath`; the file can contain JSON or YAML, compatible with the `KustomizeGeneratorOptions` struct. +By default, the specified kustomization cannot reference files or paths on `fsys` outside `kustomizationPath`. +By default, all `.yaml` or `.yml` files in `kustomizationPath`, and its subdirectories, are subject to templating, and are considered if a `kustomization.yaml` is auto-generated. It is possible to exclude certain files from templating by creating a file `.component-ignore` in `kustomizationPath`; this `.component-ignore` file uses the common `.gitignore` syntax. Note that excluded files are still visible to the `readFile` template function. Furthermore, additional file outside the `kustomizationPath` can be referenced if the according paths are declared in `.component-config.yaml` as: +- `includedKustomizations: []string`: a list of directory paths relative to `kustomizationPath`; targeted directories are treated as own components, rendered with the including component's parameters (values), and then supplied to kustomize at the identical path; recursive inclusions are possible, but must not lead to cycles (there is a circuit breaking logic that will fail the generator in case of cycles). +- `includedFiles: []string`: a list of paths relative to `kustomizationPath` (single files or directories); all referenced files (recursively in case a directory is specified) can be used with `readFile`. -As of now, the specified kustomization must not reference files or paths outside `kustomizationPath`. Remote references are generally not supported. -By default, all `.yaml` or `.yml` files in `kustomizationPath`, and its subdirectories, are subject to templating, and are considered if a `kustomization.yaml` -is auto-generated. It is possible to exclude certain files by creating a file `.component-ignore` in `kustomizationPath`; this `.component-ignore` file uses -the common `.gitignore` syntax. \ No newline at end of file +Finally, note that remote references are not supported at all. \ No newline at end of file