kubernetes kube-apiserver源码阅读4之鉴权

2023-09-16T18:30:00Z +0800 362

文章目录

如果认证成功，那么客户端的可以标识成了一个用户了，而这个用户一般至少拥有两个信息，用户和用户组。通过这两个信息就可以给用户鉴权了。

k8s一共支持6种鉴权

const (
	// ModeAlwaysAllow is the mode to set all requests as authorized
	ModeAlwaysAllow string = "AlwaysAllow"
	// ModeAlwaysDeny is the mode to set no requests as authorized
	ModeAlwaysDeny string = "AlwaysDeny"
	// ModeABAC is the mode to use Attribute Based Access Control to authorize
	ModeABAC string = "ABAC"
	// ModeWebhook is the mode to make an external webhook call to authorize
	ModeWebhook string = "Webhook"
	// ModeRBAC is the mode to use Role Based Access Control to authorize
	ModeRBAC string = "RBAC"
	// ModeNode is an authorization mode that authorizes API requests made by kubelets.
	ModeNode string = "Node"
)

但是常用的一般就是后两种，通过下面的参数指定

kube-apiserver --authorization-mode=Node,RBAC

Authorization初始化

// cmd\kube-apiserver\app\options\options.go
func NewServerRunOptions() *ServerRunOptions {
	s := ServerRunOptions{
		Authorization:           kubeoptions.NewBuiltInAuthorizationOptions(),
    }
}

func NewBuiltInAuthorizationOptions() *BuiltInAuthorizationOptions {
	return &BuiltInAuthorizationOptions{
        // 
		Modes:                       []string{authzmodes.ModeAlwaysAllow},
		WebhookVersion:              "v1beta1",
		WebhookCacheAuthorizedTTL:   5 * time.Minute,
		WebhookCacheUnauthorizedTTL: 30 * time.Second,
		WebhookRetryBackoff:         genericoptions.DefaultAuthWebhookRetryBackoff(),
	}
}

// pkg\kubeapiserver\options\authorization.go
func (o *BuiltInAuthorizationOptions) AddFlags(fs *pflag.FlagSet) {
	fs.StringSliceVar(&o.Modes, "authorization-mode", o.Modes, ""+
		"Ordered list of plug-ins to do authorization on secure port. Comma-delimited list of: "+
		strings.Join(authzmodes.AuthorizationModeChoices, ",")+".")
}

可以看到默认情况下，如果没有指定授权默认就是都允许。

然后继续看看是怎么生成鉴权器(Authorizer)的。

// cmd\kube-apiserver\app\server.go
func buildGenericConfig() {
    // 1.
    genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer(s, genericConfig.EgressSelector, versionedInformers)
}

func BuildAuthorizer() {
    // 2.
	authorizationConfig := s.Authorization.ToAuthorizationConfig(versionedInformers)
    // 3.
	return authorizationConfig.New()
}

func (config Config) New() (authorizer.Authorizer, authorizer.RuleResolver, error) {
	var (
		authorizers   []authorizer.Authorizer
		ruleResolvers []authorizer.RuleResolver
	)

	for _, authorizationMode := range config.AuthorizationModes {
		switch authorizationMode {
        // 4.
		case modes.ModeNode:
			graph := node.NewGraph()
			node.AddGraphEventHandlers(
				graph,
				config.VersionedInformerFactory.Core().V1().Nodes(),
				config.VersionedInformerFactory.Core().V1().Pods(),
				config.VersionedInformerFactory.Core().V1().PersistentVolumes(),
				config.VersionedInformerFactory.Storage().V1().VolumeAttachments(),
			)
			nodeAuthorizer := node.NewAuthorizer(graph, nodeidentifier.NewDefaultNodeIdentifier(), bootstrappolicy.NodeRules())
			authorizers = append(authorizers, nodeAuthorizer)
			ruleResolvers = append(ruleResolvers, nodeAuthorizer)
        // 5.
		case modes.ModeRBAC:
			rbacAuthorizer := rbac.New(
				&rbac.RoleGetter{Lister: config.VersionedInformerFactory.Rbac().V1().Roles().Lister()},
				&rbac.RoleBindingLister{Lister: config.VersionedInformerFactory.Rbac().V1().RoleBindings().Lister()},
				&rbac.ClusterRoleGetter{Lister: config.VersionedInformerFactory.Rbac().V1().ClusterRoles().Lister()},
				&rbac.ClusterRoleBindingLister{Lister: config.VersionedInformerFactory.Rbac().V1().ClusterRoleBindings().Lister()},
			)
			authorizers = append(authorizers, rbacAuthorizer)
			ruleResolvers = append(ruleResolvers, rbacAuthorizer)
		default:
			return nil, nil, fmt.Errorf("unknown authorization mode %s specified", authorizationMode)
		}
	}

	return union.New(authorizers...), union.NewRuleResolvers(ruleResolvers...), nil
}

代码分解如下:

和认证差不多，也是在buildGenericConfig中初始化
将命令行参数赋值到一个Config对象，这个只是简单的复制
基于上一步的config对象创建授权器(Authorizer)
构建Node模式的授权器(Authorizer)
构建RBAC模式的授权器(Authorizer)

ModeNode

kubelet使用的是证书认证，我们可以看看它的证书信息

certinfo kubelet.pem 

--- [kubelet.pem] ---
Version: 3
Serial Number: 180212443110734332529995403152692838014
Signature Algorithm: SHA256-RSA
Type: end-entity
Issuer: CN=kubernetes
Validity
    Not Before: Jun  7 09:45:06 2023 UTC
    Not After : Jun  6 09:45:06 2024 UTC
Subject: CN=system:node:node1,O=system:nodes
DNS Names: 
IP Addresses: 
Key Usage: Digital Signature, Key Encipherment
Ext Key Usage: Client Auth
CA: false

比较重要的就是Subject字段，可以看到kubelet的用户名和用户组

用户名: system:node:node1
用户组: system:nodes

然后是请求链上的鉴权逻辑。

func WithAuthorization(handler http.Handler, a authorizer.Authorizer, s runtime.NegotiatedSerializer) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		ctx := req.Context()
		ae := request.AuditEventFrom(ctx)
		// 1.
		attributes, err := GetAuthorizerAttributes(ctx)
        // 2.
		authorized, reason, err := a.Authorize(ctx, attributes)
		// 3.
		if authorized == authorizer.DecisionAllow {
			audit.LogAnnotation(ae, decisionAnnotationKey, decisionAllow)
			audit.LogAnnotation(ae, reasonAnnotationKey, reason)
			handler.ServeHTTP(w, req)
			return
		}
		responsewriters.Forbidden(ctx, attributes, w, req, reason, s)
	})
}

// 4.
func GetAuthorizerAttributes(ctx context.Context) (authorizer.Attributes, error) {
	attribs := authorizer.AttributesRecord{}
    // 5.
	user, ok := request.UserFrom(ctx)
	if ok {
		attribs.User = user
	}
	requestInfo, found := request.RequestInfoFrom(ctx)
	attribs.ResourceRequest = requestInfo.IsResourceRequest
	attribs.Path = requestInfo.Path
	attribs.Verb = requestInfo.Verb

	attribs.APIGroup = requestInfo.APIGroup
	attribs.APIVersion = requestInfo.APIVersion
	attribs.Resource = requestInfo.Resource
	attribs.Subresource = requestInfo.Subresource
	attribs.Namespace = requestInfo.Namespace
	attribs.Name = requestInfo.Name
	return &attribs, nil
}

func (r *NodeAuthorizer) Authorize(ctx context.Context, attrs authorizer.Attributes) (authorizer.Decision, string, error) {
    // 6.
	nodeName, isNode := r.identifier.NodeIdentity(attrs.GetUser())

	// 7. 
	if attrs.IsResourceRequest() {
		requestResource := schema.GroupResource{Group: attrs.GetAPIGroup(), Resource: attrs.GetResource()}
		switch requestResource {
		case secretResource:
			return r.authorizeReadNamespacedObject(nodeName, secretVertexType, attrs)
		case configMapResource:
			return r.authorizeReadNamespacedObject(nodeName, configMapVertexType, attrs)
		case pvcResource:
			if r.features.Enabled(features.ExpandPersistentVolumes) {
				if attrs.GetSubresource() == "status" {
					return r.authorizeStatusUpdate(nodeName, pvcVertexType, attrs)
				}
			}
			return r.authorizeGet(nodeName, pvcVertexType, attrs)
            // 其他资源...
			}
			return authorizer.DecisionNoOpinion, fmt.Sprintf("disabled by feature gates %s", features.CSINodeInfo), nil
		}

	}

	// 8.
	if rbac.RulesAllow(attrs, r.nodeRules...) {
		return authorizer.DecisionAllow, "", nil
	}
	return authorizer.DecisionNoOpinion, "", nil
}

代码分解如下:

获取客户端请求的属性
依次调用启用的鉴权器进行鉴权
判断是否允许，如果可以就直接到可以到下一步了
第一步的具体实现
获取用户信息和请求信息
获取node名，如果不是请求方不是node，那么就不继续了，通过组名判断是否是Node请求
针对特定资源的一些请求鉴权
非特定资源的规则鉴权，可以看到这里使用的也是rbac的方式

node的鉴权分为两个部分，一些特定资源，比如secrets，configma之类有特定的快捷鉴权逻辑，这里就不展开了，这里主要看通用的部分。可以看到鉴权中比较重要的是鉴权规则(rules)。

这里简单的看看node节点的规则列表

仅截取了一小部分

// 1.
nodeAuthorizer := node.NewAuthorizer(graph, nodeidentifier.NewDefaultNodeIdentifier(), bootstrappolicy.NodeRules())

func NodeRules() []rbacv1.PolicyRule {
	nodePolicyRules := []rbacv1.PolicyRule{
rbacv1helpers.NewRule("create").Groups(authenticationGroup).Resources("tokenreviews").RuleOrDie(),		rbacv1helpers.NewRule("create").Groups(authorizationGroup).Resources("subjectaccessreviews", "localsubjectaccessreviews").RuleOrDie(),
     // 其他规则
    }
	nodePolicyRules = append(nodePolicyRules, rbacv1helpers.NewRule("get", "list", "watch").Groups("node.k8s.io").Resources("runtimeclasses").RuleOrDie())
	return nodePolicyRules
}

总的来说，k8s会为工作节点默认设置很多请求规则

然后看看rbac是怎么评估规则的

func (r *NodeAuthorizer) Authorize(ctx context.Context, attrs authorizer.Attributes) (authorizer.Decision, string, error) {	
	if rbac.RulesAllow(attrs, r.nodeRules...) {
		return authorizer.DecisionAllow, "", nil
	}
}

func RulesAllow(requestAttributes authorizer.Attributes, rules ...rbacv1.PolicyRule) bool {
	for i := range rules {
		if RuleAllows(requestAttributes, &rules[i]) {
			return true
		}
	}

	return false
}

func RuleAllows(requestAttributes authorizer.Attributes, rule *rbacv1.PolicyRule) bool {
	if requestAttributes.IsResourceRequest() {
		combinedResource := requestAttributes.GetResource()
		if len(requestAttributes.GetSubresource()) > 0 {
			combinedResource = requestAttributes.GetResource() + "/" + requestAttributes.GetSubresource()
		}

		return rbacv1helpers.VerbMatches(rule, requestAttributes.GetVerb()) &&
			rbacv1helpers.APIGroupMatches(rule, requestAttributes.GetAPIGroup()) &&
			rbacv1helpers.ResourceMatches(rule, combinedResource, requestAttributes.GetSubresource()) &&
			rbacv1helpers.ResourceNameMatches(rule, requestAttributes.GetName())
	}

	return rbacv1helpers.VerbMatches(rule, requestAttributes.GetVerb()) &&
		rbacv1helpers.NonResourceURLMatches(rule, requestAttributes.GetPath())
}

上面的代码不是太复杂就不分解了，就是一条一条规则去匹配，判断请求的Verb, APIGroup, Resource, ResourceName等配置是否对的上，对的上就允许。

ModeRBAC

rbac的全称叫做Role-Based Access Control(基于角色的访问控制), 从名字上来应该就可以看到，被鉴权的主体至少需要两个属性，用户名，用户的角色(或者说用户组)，也就是说通过用户名找到对应的角色，然后通过角色找对应的权限。

初始化

rbacAuthorizer := rbac.New(
    // 1.
    &rbac.RoleGetter{Lister: config.VersionedInformerFactory.Rbac().V1().Roles().Lister()},
    &rbac.RoleBindingLister{Lister: config.VersionedInformerFactory.Rbac().V1().RoleBindings().Lister()},
    &rbac.ClusterRoleGetter{Lister: config.VersionedInformerFactory.Rbac().V1().ClusterRoles().Lister()},
    &rbac.ClusterRoleBindingLister{Lister: config.VersionedInformerFactory.Rbac().V1().ClusterRoleBindings().Lister()},
)

func New() *RBACAuthorizer {
    // 2.
	authorizer := &RBACAuthorizer{
		authorizationRuleResolver: rbacregistryvalidation.NewDefaultRuleResolver(
			roles, roleBindings, clusterRoles, clusterRoleBindings,
		),
	}
	return authorizer
}

代码分解如下

各种getter和lister，传入的对象都是informer的Lister
具体的创建过程

熟悉k8s informer的人应该对这些lister不太陌生，通过这些lister可以检索当前集群内的指定资源，这里主要是Role, RoleBinding, ClusterRole, ClusterRoleBinding。

可以预见的是，RoleBinding或者ClusterRoleBinding说明了角色绑定的角色，然后通过这个角色就能找到其被授权的资源。这里看一下default serviceaccount对应的role和rolebinding.

首先回顾一下default ServiceAccount的userInfo

func (sa *ServiceAccountInfo) UserInfo() user.Info {
	info := &user.DefaultInfo{
        // ServiceAccountUsernamePrefix + namespace + ServiceAccountUsernameSeparator + name
        // system:serviceaccount:default:default
		Name:   MakeUsername(sa.Namespace, sa.Name),
		UID:    sa.UID,
        // system:serviceaccounts
        // system:serviceaccounts:default
		Groups: MakeGroupNames(sa.Namespace),
	}
	return info
}
# 也就是下面的数据
{
  Name: "system:serviceaccount:default:default"
  Groups: []string{"system:serviceaccounts", "system:serviceaccounts:default"}
}

也就是说default ServiceAccount属于"system:serviceaccounts", “system:serviceaccounts:default"这两个用户组。

通过查询clusterrolebinding我们知道它对应的clusterrolebinding，clusterrole结果如下

可以通过kubectl get clusterrolebinding -o yaml|grep serviceaccounts -B 8这行命令去寻找

ClusterRoleBinding

kubectl get clusterrolebinding  system:service-account-issuer-discovery -o yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
  # 省略metadata
  name: system:service-account-issuer-discovery
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:service-account-issuer-discovery
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:serviceaccounts

ClusterRole

kubectl get clusterrole system:service-account-issuer-discovery -o yaml

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  # 省略metadata
  name: system:service-account-issuer-discovery
rules:
- nonResourceURLs:
  - /.well-known/openid-configuration
  - /openid/v1/jwks
  verbs:
  - get

可以看到default ServiceAccount的权限还是很大的可以get所有资源。

鉴权

然后继续看看其对应的鉴权逻辑

func (r *RBACAuthorizer) Authorize(ctx context.Context, requestAttributes authorizer.Attributes) (authorizer.Decision, string, error) {
	ruleCheckingVisitor := &authorizingVisitor{requestAttributes: requestAttributes}
	// 1.
	r.authorizationRuleResolver.VisitRulesFor(requestAttributes.GetUser(), requestAttributes.GetNamespace(), ruleCheckingVisitor.visit)
	if ruleCheckingVisitor.allowed {
		return authorizer.DecisionAllow, ruleCheckingVisitor.reason, nil
	}
}

func (r *DefaultRuleResolver) VisitRulesFor(user user.Info, namespace string, visitor func(source fmt.Stringer, rule *rbacv1.PolicyRule, err error) bool) {
	if clusterRoleBindings, err := r.clusterRoleBindingLister.ListClusterRoleBindings(); err != nil {
		if !visitor(nil, nil, err) {
			return
		}
	} else {
		sourceDescriber := &clusterRoleBindingDescriber{}
		for _, clusterRoleBinding := range clusterRoleBindings {
            // 2.
			subjectIndex, applies := appliesTo(user, clusterRoleBinding.Subjects, "")
			if !applies {
				continue
			}
            // 3.
			rules, err := r.GetRoleReferenceRules(clusterRoleBinding.RoleRef, "")
			sourceDescriber.binding = clusterRoleBinding
			sourceDescriber.subject = &clusterRoleBinding.Subjects[subjectIndex]
			for i := range rules {
                // 4.
				if !visitor(sourceDescriber, &rules[i], nil) {
					return
				}
			}
		}
	}

	// rolebinding的逻辑处理逻辑，和clusterrolebinding差不多
}

// 5.
func (v *authorizingVisitor) visit(source fmt.Stringer, rule *rbacv1.PolicyRule, err error) bool {
    // 6.
	if rule != nil && RuleAllows(v.requestAttributes, rule) {
		v.allowed = true
		v.reason = fmt.Sprintf("RBAC: allowed by %s", source.String())
		return false
	}
	if err != nil {
		v.errors = append(v.errors, err)
	}
	return true
}

代码分解如下:

将鉴权的权限委托给authorizationRuleResolver
匹配用户组是否能够匹配上，如果对的上就说明找到了对应的clusterrolebinding
基于上一步的clusterrolebinding找到对应的role，以及role所关联的规则
一条规则一条规则的匹配。
vistor的具体实现
使用RuleAllows匹配规则

注: 没有指定资源名就是允许所有资源名。

总结

至此，apiserver请求前的部分几乎看完了，还差准入，下一节看准入(Admission)的代码。

Authorization初始化

ModeNode

ModeRBAC

初始化

鉴权

总结

参考链接