From 47c1be74af29ca00f724887f893be2569ac42800 Mon Sep 17 00:00:00 2001 From: Santosh Kumar Gajawada Date: Wed, 13 Dec 2023 13:12:03 +0530 Subject: [PATCH] PB-4958: v1 --- pkg/apis/stork/v1alpha1/applicationrestore.go | 17 +- .../controllers/applicationclone.go | 2 +- .../controllers/applicationrestore.go | 82 ++++++ .../controllers/resourcetransformation.go | 12 +- pkg/resourcecollector/resourcecollector.go | 27 ++ .../resourcetransformation.go | 250 ++++++++++++------ 6 files changed, 293 insertions(+), 97 deletions(-) diff --git a/pkg/apis/stork/v1alpha1/applicationrestore.go b/pkg/apis/stork/v1alpha1/applicationrestore.go index 251f9692e1..b4ffa0772a 100644 --- a/pkg/apis/stork/v1alpha1/applicationrestore.go +++ b/pkg/apis/stork/v1alpha1/applicationrestore.go @@ -24,14 +24,15 @@ type ApplicationRestore struct { // ApplicationRestoreSpec is the spec used to restore applications type ApplicationRestoreSpec struct { - BackupName string `json:"backupName"` - BackupLocation string `json:"backupLocation"` - NamespaceMapping map[string]string `json:"namespaceMapping"` - ReplacePolicy ApplicationRestoreReplacePolicyType `json:"replacePolicy"` - IncludeOptionalResourceTypes []string `json:"includeOptionalResourceTypes"` - IncludeResources []ObjectInfo `json:"includeResources"` - StorageClassMapping map[string]string `json:"storageClassMapping"` - RancherProjectMapping map[string]string `json:"rancherProjectMapping"` + BackupName string `json:"backupName"` + BackupLocation string `json:"backupLocation"` + NamespaceMapping map[string]string `json:"namespaceMapping"` + ReplacePolicy ApplicationRestoreReplacePolicyType `json:"replacePolicy"` + IncludeOptionalResourceTypes []string `json:"includeOptionalResourceTypes"` + IncludeResources []ObjectInfo `json:"includeResources"` + StorageClassMapping map[string]string `json:"storageClassMapping"` + RancherProjectMapping map[string]string `json:"rancherProjectMapping"` + ResourceTransformationTemplate string `json:"resourceTransformationTemplate"` } // ApplicationRestoreReplacePolicyType is the replace policy for the application restore diff --git a/pkg/applicationmanager/controllers/applicationclone.go b/pkg/applicationmanager/controllers/applicationclone.go index 18b9df5259..5eca633a65 100644 --- a/pkg/applicationmanager/controllers/applicationclone.go +++ b/pkg/applicationmanager/controllers/applicationclone.go @@ -547,7 +547,7 @@ func (a *ApplicationCloneController) prepareResources( clone.Spec.IncludeOptionalResourceTypes, nil, &opts, - "", "", + "", "", nil, ) if err != nil { return nil, err diff --git a/pkg/applicationmanager/controllers/applicationrestore.go b/pkg/applicationmanager/controllers/applicationrestore.go index c79785cb02..6c04950ff4 100644 --- a/pkg/applicationmanager/controllers/applicationrestore.go +++ b/pkg/applicationmanager/controllers/applicationrestore.go @@ -163,13 +163,81 @@ func (a *ApplicationRestoreController) verifyNamespaces(restore *storkapi.Applic func (a *ApplicationRestoreController) createNamespaces(backup *storkapi.ApplicationBackup, backupLocation string, restore *storkapi.ApplicationRestore) error { + logrus.Info("Starting createNamespaces") var namespaces []*v1.Namespace + // rtTemplate := &storkapi.ResourceTransformation{} + // a.client.Get(context.Background(), types.NamespacedName{Name: restore.Spec.ResourceTransformationTemplate, Namespace: kdmputils.AdminNamespace}, rtTemplate) + + var gvkResourceTransform map[schema.GroupVersionKind][]storkapi.TransformSpecs + if restore.Spec.ResourceTransformationTemplate != "" { + var err error + gvkResourceTransform, err = resourcecollector.GetGVKToTransformSpecMapper(restore.Spec.ResourceTransformationTemplate, kdmputils.AdminNamespace) + if err != nil { + log.ApplicationRestoreLog(restore). + Warnf("Unable to get transformation spec from :%s, skipping transformation for this restore, err: %v", restore.Spec.ResourceTransformationTemplate, err) + return err + } + } + + logrus.Info("gvkResourceTransform: ", gvkResourceTransform) + nsData, err := a.downloadObject(backup, backupLocation, restore.Namespace, nsObjectName, true) if err != nil { return err } + logrus.Info("Before nsdata: ", string(nsData)) + + if gvkResourceTransform != nil { + objects := make([]*unstructured.Unstructured, 0) + if err = json.Unmarshal(nsData, &namespaces); err != nil { + return err + } + + for _, ns := range namespaces { + var object unstructured.Unstructured + content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(ns) + if err != nil { + logrus.Errorf("Error while DefaultUnstructuredConverter.ToUnstructured: %v", err) + } + object.SetUnstructuredContent(content) + objects = append(objects, &object) + } + + logrus.Info("one: objects: ", objects) + // gvk, err := resourcecollector.GetGVK(nsData) + // if err != nil { + // return err + // } + gvk := schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Namespace", + } + logrus.Info("two: gvk: ", gvk) + + if transforms, ok := gvkResourceTransform[gvk]; ok { + for _, object := range objects { + for _, transform := range transforms { + if resourcecollector.IsObjectLabelsMatched(object.UnstructuredContent(), transform.Selectors) { + resourcecollector.TransformObject(object, transform) + } + } + } + } + logrus.Info("three: transform done ") + + //v1.SchemeGroupVersion.WithKind() + nsData, err = json.Marshal(objects) + if err != nil { + logrus.Error("error while marshaling the unstructed objects to []byte: err", err) + return err + } + } + + logrus.Info("After nsdata: ", string(nsData)) + rancherProjectMapping := getRancherProjectMapping(restore) if nsData != nil { if err = json.Unmarshal(nsData, &namespaces); err != nil { @@ -683,6 +751,7 @@ func (a *ApplicationRestoreController) restoreVolumes(restore *storkapi.Applicat &opts, restore.Spec.BackupLocation, restore.Namespace, + nil, ) if err != nil { return err @@ -1593,6 +1662,18 @@ func (a *ApplicationRestoreController) applyResources( break } + var gvkResourceTransform map[schema.GroupVersionKind][]storkapi.TransformSpecs + log.ApplicationRestoreLog(restore).Info("restore.Spec.ResourceTransformationTemplate value:", restore.Spec.ResourceTransformationTemplate) + if restore.Spec.ResourceTransformationTemplate != "" { + var err error + gvkResourceTransform, err = resourcecollector.GetGVKToTransformSpecMapper(restore.Spec.ResourceTransformationTemplate, kdmputils.AdminNamespace) + if err != nil { + log.ApplicationRestoreLog(restore). + Warnf("Unable to get transformation spec from :%s, skipping transformation for this restore, err: %v", restore.Spec.ResourceTransformationTemplate, err) + return err + } + } + // This channel listens on two values if the CR's time stamp to be updated or to quit the go routine startTime := time.Now() for _, o := range objects { @@ -1613,6 +1694,7 @@ func (a *ApplicationRestoreController) applyResources( &opts, restore.Spec.BackupLocation, restore.Namespace, + gvkResourceTransform, ) if err != nil { return err diff --git a/pkg/migration/controllers/resourcetransformation.go b/pkg/migration/controllers/resourcetransformation.go index a68655be36..761e996b71 100644 --- a/pkg/migration/controllers/resourcetransformation.go +++ b/pkg/migration/controllers/resourcetransformation.go @@ -62,6 +62,11 @@ func (r *ResourceTransformationController) Init(mgr manager.Manager) error { return controllers.RegisterTo(mgr, ResourceTransformationControllerName, r, &stork_api.ResourceTransformation{}) } +const ( + rtCreatedForAnnotationKey = "portworx.io/CreatedFor" + rtCreatedForRestoreAnnotationValue = "restore" +) + // Reconcile manages ResourceTransformation resources. func (r *ResourceTransformationController) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { resourceTransformation := &stork_api.ResourceTransformation{} @@ -121,6 +126,9 @@ func (r *ResourceTransformationController) handle(ctx context.Context, transform return nil } transform.Status.Status = stork_api.ResourceTransformationStatusInProgress + if value, ok := transform.GetAnnotations()[rtCreatedForAnnotationKey]; ok && value == rtCreatedForRestoreAnnotationValue { + transform.Status.Status = stork_api.ResourceTransformationStatusReady + } if err = r.client.Update(ctx, transform); err != nil { return err } @@ -154,9 +162,7 @@ func (r *ResourceTransformationController) validateSpecPath(transform *stork_api if err != nil { return err } - if !resourcecollector.GetSupportedK8SResources(kind, []string{}) { - return fmt.Errorf("unsupported resource kind for transformation: %s", kind) - } + for _, path := range spec.Paths { // TODO: this can be validated via CRDs as well, when we have defined schema // for stork crds diff --git a/pkg/resourcecollector/resourcecollector.go b/pkg/resourcecollector/resourcecollector.go index e8d9445b5f..0d20770cde 100644 --- a/pkg/resourcecollector/resourcecollector.go +++ b/pkg/resourcecollector/resourcecollector.go @@ -1131,7 +1131,9 @@ func (r *ResourceCollector) PrepareResourceForApply( opts *Options, backuplocationName string, backuplocationNamespace string, + gvkResourceTransform map[schema.GroupVersionKind][]stork_api.TransformSpecs, ) (bool, error) { + logrus.Info("Starting PrepareResourceForApply") objectType, err := meta.TypeAccessor(object) if err != nil { return false, err @@ -1158,6 +1160,31 @@ func (r *ResourceCollector) PrepareResourceForApply( // Update the namespace of the object, will be no-op for clustered resources metadata.SetNamespace(val) } + + logrus.Infof("gvkResourceTransform: %v", gvkResourceTransform) + if gvkResourceTransform != nil { + gv, err := schema.ParseGroupVersion(objectType.GetAPIVersion()) + logrus.Infof("Parse groupVersion %v", gv) + if err != nil { + logrus.Error("Error while parsing the group version for the object: ", object) + } + gvk := gv.WithKind(objectType.GetKind()) + logrus.Infof("groupVersion %v with kind %v is gvk: %v", gv, objectType.GetKind(), gvk) + + if transforms, ok := gvkResourceTransform[gvk]; ok { + logrus.Infof("[]stork_api.TransformSpecs %v for gvk %v", transforms, gvk) + for _, transform := range transforms { + logrus.Infof("TransformSpec %v for gvk %v", transform, gvk) + if IsObjectLabelsMatched(object.UnstructuredContent(), transform.Selectors) { + err := TransformObject(object, transform) + if err != nil { + logrus.Error("Error Transforming the Object") + } + } + } + } + } + switch objectType.GetKind() { case "Job": if slice.ContainsString(optionalResourceTypes, "job", strings.ToLower) || diff --git a/pkg/resourcecollector/resourcetransformation.go b/pkg/resourcecollector/resourcetransformation.go index c5eb2df83c..9d7c9139e1 100644 --- a/pkg/resourcecollector/resourcetransformation.go +++ b/pkg/resourcecollector/resourcetransformation.go @@ -1,6 +1,7 @@ package resourcecollector import ( + "encoding/json" "fmt" "strconv" "strings" @@ -9,10 +10,72 @@ import ( storkops "github.com/portworx/sched-ops/k8s/stork" "github.com/sirupsen/logrus" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" ) +func GetGVK(Data []byte) (schema.GroupVersionKind, error) { + metadata := []metav1.TypeMeta{} + if err := json.Unmarshal(Data, &metadata); err != nil { + return schema.GroupVersionKind{}, err + } + gVersion, err := schema.ParseGroupVersion(metadata[0].APIVersion) + if err != nil { + return schema.GroupVersionKind{}, err + } + return gVersion.WithKind(metadata[0].Kind), nil +} + +func IsObjectLabelsMatched(content map[string]interface{}, selector map[string]string) bool { + logrus.Info("Starting IsObjectLabelsMatched") + objectLabels, _, err := unstructured.NestedStringMap(content, "metadata", "labels") + if err != nil { + logrus.Error("error finding the metadata.labels from the unstructed content: ", content) + } + logrus.Infof("ObjectLabels %v, selector %v,output %v", objectLabels, selector, labels.SelectorFromSet(selector).Matches(labels.Set(objectLabels))) + return labels.SelectorFromSet(selector).Matches(labels.Set(objectLabels)) + // labels.Equals(objectLabels, selector) +} + +// GetGVKToTransformSpecMapper {GVK: []TransformSpecs{}} +func GetGVKToTransformSpecMapper(transformName string, namespace string) (map[schema.GroupVersionKind][]stork_api.TransformSpecs, error) { + logrus.Info("Starting GetGVKToTransformSpecMapper") + gvkResourceTransform := make(map[schema.GroupVersionKind][]stork_api.TransformSpecs) + if transformName == "" || namespace == "" { + logrus.Errorf("empty name received for resource transformation/namespace") + return gvkResourceTransform, fmt.Errorf("empty name received for resource transformation/namespace") + } + + resp, err := storkops.Instance().GetResourceTransformation(transformName, namespace) + if err != nil { + // current namespace does not have any transform CR + // skip it from map + if errors.IsNotFound(err) { + logrus.Errorf("resource not found for resource transformation/namespace: %s,%s", transformName, namespace) + return gvkResourceTransform, fmt.Errorf("resource not found for resource transformation/namespace: %s,%s", transformName, namespace) + } + logrus.Errorf("unable to get resource transfomration specs %s/%s, err: %v", namespace, transformName, err) + return nil, fmt.Errorf("unable to get resource transfomration specs %s/%s, err: %v", namespace, transformName, err) + } + + logrus.Infof("resp: %+v", *resp) + + for _, spec := range resp.Spec.Objects { + if gvk, err := getGVK(spec.Resource); err != nil { + logrus.Error("Error getGVK: ", err) + return nil, err + } else { + logrus.Infof("update gvkResourceTransform for gvk %v with spec %v", gvk, spec) + gvkResourceTransform[gvk] = append(gvkResourceTransform[gvk], spec) + } + } + logrus.Info("Completed GetGVKToTransformSpecMapper") + return gvkResourceTransform, nil +} + // Since we collect all resources from required migration namespace at once // getResourcePatch creates map of namespace: {kind: []resourceinfo{}} // to get transform spec for matching resources @@ -51,100 +114,108 @@ func TransformResources( ) error { for _, patch := range resPatch { if patch.Name == objName && patch.Namespace == objNamespace { - content := object.UnstructuredContent() - for _, path := range patch.Specs.Paths { - switch path.Operation { - case stork_api.AddResourcePath: - value, err := getNewValueForPath(path.Value, path.Type) - if err != nil { - logrus.Errorf("Unable to parse the Value for the type %s specified, path %s on resource kind: %s/,%s/%s, err: %v", path.Type, path, patch.Kind, patch.Namespace, patch.Name, err) - return err - } - if path.Type == stork_api.KeyPairResourceType { - updateMap := value.(map[string]string) - err := unstructured.SetNestedStringMap(content, updateMap, strings.Split(path.Path, ".")...) - if err != nil { - logrus.Errorf("Unable to apply patch path %s on resource kind: %s/,%s/%s, err: %v", path, patch.Kind, patch.Namespace, patch.Name, err) - return err - } - } else if path.Type == stork_api.SliceResourceType { - updateSlice := value.([]string) - err := unstructured.SetNestedStringSlice(content, updateSlice, strings.Split(path.Path, ".")...) - if err != nil { - logrus.Errorf("Unable to apply patch path %s on resource kind: %s/,%s/%s, err: %v", path, patch.Kind, patch.Namespace, patch.Name, err) - return err - } - } else { - err := unstructured.SetNestedField(content, value, strings.Split(path.Path, ".")...) - if err != nil { - logrus.Errorf("Unable to perform operation %s on path %s on resource kind: %s/,%s/%s, err: %v", path.Operation, path, patch.Kind, patch.Namespace, patch.Name, err) - return err - } - } - - case stork_api.DeleteResourcePath: - unstructured.RemoveNestedField(content, strings.Split(path.Path, ".")...) - logrus.Debugf("Removed patch path %s on resource kind: %s/,%s/%s", path, patch.Kind, patch.Namespace, patch.Name) - - case stork_api.ModifyResourcePathValue: - var value interface{} - if path.Type == stork_api.KeyPairResourceType { - currMap, _, err := unstructured.NestedMap(content, strings.Split(path.Path, ".")...) - if err != nil || len(currMap) == 0 { - return fmt.Errorf("unable to find spec path, err: %v", err) - } - mapList := strings.Split(path.Value, ",") - for _, val := range mapList { - keyPair := strings.Split(val, ":") - if len(keyPair) != 2 { - return fmt.Errorf("invalid keypair value format :%s", keyPair) - } - currMap[keyPair[0]] = keyPair[1] - } - value = currMap - } else if path.Type == stork_api.SliceResourceType { - currList, _, err := unstructured.NestedSlice(content, strings.Split(path.Path, ".")...) - if err != nil { - return fmt.Errorf("unable to find spec path, err: %v", err) - } - arrList := strings.Split(path.Value, ",") - for _, val := range arrList { - currList = append(currList, val) - } - value = currList - } else { - var err error - value, err = getNewValueForPath(path.Value, path.Type) - if err != nil { - logrus.Errorf("Unable to parse the Value for the type %s specified, path %s on resource kind: %s/,%s/%s, err: %v", path.Type, path, patch.Kind, patch.Namespace, patch.Name, err) - return err - } - } - err := unstructured.SetNestedField(content, value, strings.Split(path.Path, ".")...) - if err != nil { - logrus.Errorf("Unable to perform operation %s on path %s on resource kind: %s/,%s/%s, err: %v", path.Operation, path, patch.Kind, patch.Namespace, patch.Name, err) - return err - } - } + if err := TransformObject(object, patch.Specs); err != nil { + logrus.Errorf("Unable to Transform Object %s/%s, on resource kind: %s, err: %v", patch.Kind, patch.Namespace, patch.Name, err) } - // lets add annotation saying this resource has been transformed by migration/restore - // controller before applying - // set migration annotations - annotations, found, err := unstructured.NestedStringMap(content, "metadata", "annotations") + } + } + return nil +} + +func TransformObject(object runtime.Unstructured, spec stork_api.TransformSpecs) error { + logrus.Info("Started TransformObject") + content := object.UnstructuredContent() + for _, path := range spec.Paths { + switch path.Operation { + case stork_api.AddResourcePath: + value, err := getNewValueForPath(path.Value, path.Type) if err != nil { + logrus.Errorf("Unable to parse the Value for the type %s specified, path %s on resource: %s, err: %v", path.Type, path, spec.Resource, err) return err } - if !found { - annotations = make(map[string]string) + if path.Type == stork_api.KeyPairResourceType { + updateMap := value.(map[string]string) + err := unstructured.SetNestedStringMap(content, updateMap, strings.Split(path.Path, ".")...) + if err != nil { + logrus.Errorf("Unable to apply patch path %s on resource: %s, err: %v", path, spec.Resource, err) + return err + } + } else if path.Type == stork_api.SliceResourceType { + updateSlice := value.([]string) + err := unstructured.SetNestedStringSlice(content, updateSlice, strings.Split(path.Path, ".")...) + if err != nil { + logrus.Errorf("Unable to apply patch path %s on resource: %s, err: %v", path, spec.Resource, err) + return err + } + } else { + err := unstructured.SetNestedField(content, value, strings.Split(path.Path, ".")...) + if err != nil { + logrus.Errorf("Unable to perform operation %s on path %s on resource: %s, err: %v", path.Operation, path, spec.Resource, err) + return err + } + } + + case stork_api.DeleteResourcePath: + unstructured.RemoveNestedField(content, strings.Split(path.Path, ".")...) + logrus.Debugf("Removed patch path %s on resource: %s", path, spec.Resource) + + case stork_api.ModifyResourcePathValue: + var value interface{} + if path.Type == stork_api.KeyPairResourceType { + currMap, _, err := unstructured.NestedMap(content, strings.Split(path.Path, ".")...) + if err != nil || len(currMap) == 0 { + return fmt.Errorf("unable to find spec path, err: %v", err) + } + mapList := strings.Split(path.Value, ",") + for _, val := range mapList { + keyPair := strings.Split(val, ":") + if len(keyPair) != 2 { + return fmt.Errorf("invalid keypair value format :%s", keyPair) + } + currMap[keyPair[0]] = keyPair[1] + } + value = currMap + } else if path.Type == stork_api.SliceResourceType { + currList, _, err := unstructured.NestedSlice(content, strings.Split(path.Path, ".")...) + if err != nil { + return fmt.Errorf("unable to find spec path, err: %v", err) + } + arrList := strings.Split(path.Value, ",") + for _, val := range arrList { + currList = append(currList, val) + } + value = currList + } else { + var err error + value, err = getNewValueForPath(path.Value, path.Type) + if err != nil { + logrus.Errorf("Unable to parse the Value for the type %s specified, path %s on resource: %s, err: %v", path.Type, path, spec.Resource, err) + return err + } } - annotations[TransformedResourceName] = "true" - if err := unstructured.SetNestedStringMap(content, annotations, "metadata", "annotations"); err != nil { + err := unstructured.SetNestedField(content, value, strings.Split(path.Path, ".")...) + if err != nil { + logrus.Errorf("Unable to perform operation %s on path %s on resource: %s, err: %v", path.Operation, path, spec.Resource, err) return err } - object.SetUnstructuredContent(content) - logrus.Infof("Updated resource of kind %v with patch , resource: %v", patch.Kind, object) } } + // lets add annotation saying this resource has been transformed by migration/restore + // controller before applying + // set migration annotations + annotations, found, err := unstructured.NestedStringMap(content, "metadata", "annotations") + if err != nil { + return err + } + if !found { + annotations = make(map[string]string) + } + annotations[TransformedResourceName] = "true" + if err := unstructured.SetNestedStringMap(content, annotations, "metadata", "annotations"); err != nil { + return err + } + object.SetUnstructuredContent(content) + logrus.Infof("Updated resource %s with patch , object: %v", spec.Resource, object) return nil } @@ -177,3 +248,12 @@ func getNewValueForPath(oldVal string, valType stork_api.ResourceTransformationV } return updatedValue, err } + +// return group,version,kind from give resource type +func getGVK(resource string) (schema.GroupVersionKind, error) { + gvk := strings.Split(resource, "/") + if len(gvk) != 3 { + return schema.GroupVersionKind{}, fmt.Errorf("invalid resource kind :%s", resource) + } + return schema.GroupVersionKind{Group: gvk[0], Version: gvk[1], Kind: gvk[2]}, nil +}