spring-cloud-kubernetes学习(一) 编写第一个spring-cloud-kubernetes例子并在kubernetes中测试
在springcloud生态中，服务治理与注册中心等都有相应的组件。如eureka、hystrix，ribbon等。但是kubernetes组件也有服务发现、负载均衡的组件，我们可以借助于sping-cloud-kubernetes组件为我们提供的服务发现、负载均衡等来摈弃像eureka这样的注册中心。本文主要通过构建两个spring cloud 服务来演示spring-cloud-kubernetes组件如何做服务的发现，负载均衡等。
项目地址：https://gitee.com/quanwenz/micro-service-frame/repository/archive/first
一、首先搭建服务：
构建使用gradle,使用maven类似，只是引入方式稍微不同
1、使用gradle创建一个java工程
2、添加gradle.properties文件
这里从其他项目中拉过来的，可能有些冗余
nexusUrl=http://172.16.10.190:8081/repository/maven-public/
#nexusUrl=http://maven.aliyun.com/nexus/content/groups/public
aliyunUrl=http://maven.aliyun.com/nexus/content/groups/public
springUrl=https://repo.spring.io/libs-milestone/
#java-version
javaVersion=1.8
## dependency versions.
#springBootVersion=2.2.6.RELEASE
springBootVersion=2.3.5.RELEASE
#platformVersion=Cairo-SR7
platformVersion=Cairo-SR8
junitVersion=4.12
springCloudConfigVersion=2.0.0.RELEASE
lombokVersion=1.18.4
#micro service versions
#openFeginVersion=2.2.5.RELEASE
#springCloudKubernetesVersion=1.1.7.RELEASE
springCloudVersion=Hoxton.SR9
#springCloudCommonsVersion=2.2.6.RELEASE
#springCloudNetflixRibbonVersion=2.2.6.RELEASE
#common-redis-tools dependency versions
#jedisVersion=2.9.0
jedisVersion=3.3.0
jedis.version=${jedisVersion}
fastJsonVersion=1.2.59
lettuceCoreVersion=6.0.1.RELEASE
#common-web-tools dependency versions
servletApiVersion=4.0.1
servletApi.version=${servletApiVersion}
jsoncodeVersion=1.2.4
yuicompressorVersion=2.4.8
jsonlibVersion=2.4:jdk15
orgJsonVersion=20190722
xomVersion=1.2.5
UserAgentUtilsVersion=1.20
jacksonVersion=2.9.8
#common-tools dependency versions
htmlsuckerVersion=0.0.2
ostermillerVersion=1.07.00
commonLang3Version=3.8.1
dom4jVersion=1.6.1
poiVersion=3.17
poiOoxmlVersion=3.17
emojiJavaVersion=4.0.0
okioVersion=2.2.2
freemarkerVersion=2.3.28
bopomofo4jVersion=1.0.0
junrarVersion=4.0.0
#common-rpc-tools
sofaVersion=5.7.6
grpcVersion=1.33.1
protocVersion=3.2.0
protocGenGrpcJavaVersion=1.4.0
libthriftVersion=0.13.0
#org.apache.xmlbeans.version=2.3.0
commonsIoVersion=2.6
codecVersion=1.14
staxVersion=1.0.1
commonsBeanutilsVersion=1.9.3
itextAsianVersion=5.2.0
itextVersion=2.1.7
zxingCoreVersion=3.2.1
apacheAntVersion=1.7.1
ip2regionVersion=1.7
htoolVersion=4.1.15
cglibVersion=3.2.8
guavaVersion=20.0
pingyin4jVersion=2.5.1
itextpdfVersion=5.2.0
lowagieVersion=2.1.7
zxingVersion=3.2.1
antVersion=1.7.1
#biz-base biz-mp dependency versions
#mybatisStaterVersion=1.3.2
#mysqlVersion=5.1.47
mysqlVersion=8.0.15
pagehelperVersion=1.2.5
druidVersion=1.1.16
okhttp.version=3.10.0
freemarker.version=2.3.28
caffeine.version=2.6.2
gson.version=2.8.5
jwtVersion=3.3.0
swaggerVersion=3.0.0
nettyVersion=4.1.26.Final
asciidoctorVersion=1.5.3
asciidoctorPDFVersion=1.5.0-alpha.10.1
swagger2markupDocVersion=1.3.3
userAgentUtilVersion=1.2.4
mybatis_plus_boot_starter_version=3.3.1.tmp
velocity_engine_core_version=2.0
jsoupVersion=1.11.3
logExpansionVersion=0.0.2.RELEASE
rsaEncryVersion=1.0.1.RELEASE
knife4jVersion=2.0.1
kaptchaVersion=2.3.2
screwCoreVersion=1.0.5
#biz-jpa
javaxValidationVersion=2.0.1.Final
#log-expansion
shadowVersion=5.0.0
#javassist
javassistVersion=3.25.0-GA
hostMachineIp=192.168.100.88
gradle_docker_version=1.2
#common-k8s-tools
kubernetesClientVersion=4.6.1
#springboot-admin
bootAdminVersion=2.2.2
#rule-engine
aviatorVersion=4.2.93、创建三个模块
service-api、product-test-prop-service、consumer-test-prop-service其中service-api定义Fegin的API
product-test-prop-service模拟一个提供者服务
consumer-test-prop-service模拟一个消费者服务
4、service-api中build.xml
apply plugin: 'java'
dependencies {
compile "org.springframework.cloud:spring-cloud-starter-openfeign"
// compile group: 'javax.servlet', name: 'javax.servlet-api', version: "${servletApiVersion}"
}5、service-api中编写接口和降级处理
接口：
/**
* 测试接口
*
* @author zhuquanwen
* @vesion 1.0
* @date 2020/12/6 9:59
* @since jdk1.8
*/
@Component
@FeignClient(name="${feign.product-prop.name:product-test-service}",fallback = PropertyClientFallback.class)
public interface PropertyClient {
@GetMapping(value="properties")
List<String> getProperties();
}上面的FeignClient注解中的name使用和后面kubernetes中对应service一样的名字
降级处理：
/**
*
*
* @author zhuquanwen
* @vesion 1.0
* @date 2020/12/6 10:05
* @since jdk1.8
*/
@Component
public class PropertyClientFallback implements PropertyClient {
@Override
public List<String> getProperties() {
return new ArrayList<String>();
}
}6、product-test-prop-service中build.xml
注意引入了service-api模块
apply plugin: 'java'
dependencies {
compile project(":service-api")
compile "org.springframework.boot:spring-boot-starter-web"
compile "org.springframework.boot:spring-boot-starter-actuator"
compile group: 'org.springframework.boot', name: 'spring-boot-starter-undertow'
}7、product-test-prop-service的接口实现
/**
*
*
* @author zhuquanwen
* @vesion 1.0
* @date 2020/12/6 10:43
* @since jdk1.8
*/
@RestController
@RequestMapping
public class PropertyController implements PropertyClient {
@GetMapping("properties")
public List<String> getProperties(){
ArrayList<String> properties = new ArrayList<>();
properties.add("properties1");
properties.add("properties2");
return properties;
}
}8、product-test-prop-service的application.properties配置文件
server.port=80809、product-test-prop-service的启动类
/**
*
* @author zhuquanwen
* @vesion 1.0
* @date 2020/12/6 11:12
* @since jdk1.8
*/
@SpringBootApplication
//@EnableFeignClients
public class Start {
public static void main(String[] args) {
SpringApplication.run(Start.class, args);
}
}10、consumer-test-prop-service中build.xml
apply plugin: 'java'
dependencies {
compile project(":service-api")
compile "org.springframework.cloud:spring-cloud-kubernetes-core"
compile "org.springframework.cloud:spring-cloud-kubernetes-discovery"
compile "org.springframework.cloud:spring-cloud-starter-kubernetes-ribbon"
compile "org.springframework.cloud:spring-cloud-commons"
compile "org.springframework.cloud:spring-cloud-starter-netflix-ribbon"
compile "org.springframework.cloud:spring-cloud-starter-netflix-hystrix"
compile "org.springframework.boot:spring-boot-starter-web"
compile group: 'org.springframework.boot', name: 'spring-boot-starter-undertow'
}11、consumer-test-prop-service中调用类
/**
*
*
* @author zhuquanwen
* @vesion 1.0
* @date 2020/12/6 11:00
* @since jdk1.8
*/
@RestController
@RequestMapping
public class ConsumerPropController {
@Resource
private PropertyClient propertyClient;
@GetMapping("properties")
public List<String> getProductProperties(){
return propertyClient.getProperties();
}
}12、consumer-test-prop-service中application.yml配置文件
其中KubernetesNamespace配置为后面使用的kubernetes的命名空间
server:
port: 8080
spring:
application:
name: consumer-test-prop-service
product-test-prop-service:
ribbon:
KubernetesNamespace: default
backend:
ribbon:
eureka:
enabled: false
client:
enabled: true
ServerListRefreshInterval: 5000
hystrix:
command:
BackendCall:
execution:
isolation:
thread:
timeoutInMilliseconds: 5000
threadpool:
BackendCallThread:
coreSize: 5
feign:
hystrix:
enabled: true13、consumer-test-prop-service中启动类
/**
*
* @author zhuquanwen
* @vesion 1.0
* @date 2020/12/6 11:12
* @since jdk1.8
*/
@SpringBootApplication
@Configuration
@ServletComponentScan //自动扫描serletBean
@ComponentScan(basePackages = {"com.iscas"})
@EnableDiscoveryClient
@EnableFeignClients(basePackages = {"com.iscas"})
//@EnableFeignClients(basePackageClasses = {PropertyClient.class, PropertyClientFallback.class})
public class Start {
public static void main(String[] args) {
SpringApplication springApplication = new SpringApplication(Start.class);
springApplication.run(args);
}
}OK,现在我们把项目搭建完了，启动两个启动类(注意端口一样的，可以把生产者的端口换一下)
访问http://localhost:8080/properties在浏览器返回了“[]”，没有返回“[“properties1”,“properties2”]”，这是因为没有运行在k8s中，没有服务发现，降级处理了。
控制台的报错也能证明
2020-12-06 16:08:09.876 WARN 5356 --- [-test-service-1] s.c.a.AnnotationConfigApplicationContext : Exception encountered during context initialization - cancelling refresh attempt: org.springframework.beans.factory.UnsatisfiedDependencyException: Error creating bean with name 'ribbonLoadBalancingHttpClient' defined in org.springframework.cloud.netflix.ribbon.apache.HttpClientRibbonConfiguration: Unsatisfied dependency expressed through method 'ribbonLoadBalancingHttpClient' parameter 2; nested exception is org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'ribbonLoadBalancer' defined in org.springframework.cloud.netflix.ribbon.RibbonClientConfiguration: Bean instantiation via factory method failed; nested exception is org.springframework.beans.BeanInstantiationException: Failed to instantiate [com.netflix.loadbalancer.ILoadBalancer]: Factory method 'ribbonLoadBalancer' threw exception; nested exception is io.fabric8.kubernetes.client.KubernetesClientException: Operation: [get] for kind: [Endpoints] with name: [product-test-service] in namespace: [null] failed.
2020-12-06 16:08:10.825 WARN 5356 --- [erListUpdater-0] c.n.l.PollingServerListUpdater : Failed one update cycle
io.fabric8.kubernetes.client.KubernetesClientException: Operation: [get] for kind: [Endpoints] with name: [product-test-service] in namespace: [null] failed.
at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:64) ~[kubernetes-client-4.10.3.jar:na]
at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:72) ~[kubernetes-client-4.10.3.jar:na]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.getMandatory(BaseOperation.java:244) ~[kubernetes-client-4.10.3.jar:na]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.get(BaseOperation.java:187) ~[kubernetes-client-4.10.3.jar:na]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.get(BaseOperation.java:79) ~[kubernetes-client-4.10.3.jar:na]
at org.springframework.cloud.kubernetes.ribbon.KubernetesEndpointsServerList.getUpdatedListOfServers(KubernetesEndpointsServerList.java:58) ~[spring-cloud-kubernetes-ribbon-1.1.7.RELEASE.jar:1.1.7.RELEASE]
at com.netflix.loadbalancer.DynamicServerListLoadBalancer.updateListOfServers(DynamicServerListLoadBalancer.java:240) ~[ribbon-loadbalancer-2.3.0.jar:2.3.0]
at com.netflix.loadbalancer.DynamicServerListLoadBalancer$1.doUpdate(DynamicServerListLoadBalancer.java:62) ~[ribbon-loadbalancer-2.3.0.jar:2.3.0]
at com.netflix.loadbalancer.PollingServerListUpdater$1.run(PollingServerListUpdater.java:116) ~[ribbon-loadbalancer-2.3.0.jar:2.3.0]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_144]
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [na:1.8.0_144]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_144]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [na:1.8.0_144]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_144]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_144]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_144]
Caused by: java.net.UnknownHostException: kubernetes.default.svc
at java.net.InetAddress.getAllByName0(InetAddress.java:1280) ~[na:1.8.0_144]
at java.net.InetAddress.getAllByName(InetAddress.java:1192) ~[na:1.8.0_144]
at java.net.InetAddress.getAllByName(InetAddress.java:1126) ~[na:1.8.0_144]
at okhttp3.Dns$1.lookup(Dns.java:40) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.connection.RouteSelector.resetNextInetSocketAddress(RouteSelector.java:185) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.connection.RouteSelector.nextProxy(RouteSelector.java:149) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.connection.RouteSelector.next(RouteSelector.java:84) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.connection.StreamAllocation.findConnection(StreamAllocation.java:214) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.connection.StreamAllocation.findHealthyConnection(StreamAllocation.java:135) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.connection.StreamAllocation.newStream(StreamAllocation.java:114) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:42) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:126) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[okhttp-3.10.0.jar:na]
at io.fabric8.kubernetes.client.utils.BackwardsCompatibilityInterceptor.intercept(BackwardsCompatibilityInterceptor.java:134) ~[kubernetes-client-4.10.3.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[okhttp-3.10.0.jar:na]
at io.fabric8.kubernetes.client.utils.ImpersonatorInterceptor.intercept(ImpersonatorInterceptor.java:68) ~[kubernetes-client-4.10.3.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[okhttp-3.10.0.jar:na]
at io.fabric8.kubernetes.client.utils.HttpClientUtils.lambda$createHttpClient$3(HttpClientUtils.java:149) ~[kubernetes-client-4.10.3.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[okhttp-3.10.0.jar:na]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[okhttp-3.10.0.jar:na]
at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:200) ~[okhttp-3.10.0.jar:na]
at okhttp3.RealCall.execute(RealCall.java:77) ~[okhttp-3.10.0.jar:na]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:490) ~[kubernetes-client-4.10.3.jar:na]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:451) ~[kubernetes-client-4.10.3.jar:na]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:416) ~[kubernetes-client-4.10.3.jar:na]
at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:397) ~[kubernetes-client-4.10.3.jar:na]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleGet(BaseOperation.java:890) ~[kubernetes-client-4.10.3.jar:na]
at io.fabric8.kubernetes.client.dsl.base.BaseOperation.getMandatory(BaseOperation.java:233) ~[kubernetes-client-4.10.3.jar:na]
... 13 common frames omitted二、将两个服务运行在kubernetes中
现在我们将两个服务打包并在kubernetes集群中运行，kubernetes的搭建过程可参考前面的文章
1、gradle将两个服务打包，上传至安装了docker服务上
2、将两个jar包打成镜像
一个jar打完再打另一个
把jar包名改为app.jar
在jar包所在目录编写Dockerfile
FROM java:8-alpine
ADD app.jar app.jar
EXPOSE 8080
ENTRYPOINT ["java","-jar", "/app.jar"]java:8-alpine 镜像可以从网上下载，多的很
运行两个命令将jar包
docker build -t 192.168.100.91:80/product-test:0.0.1 .192.168.100.91:80是我本地镜像服务的地址
推送到docker镜像服务
docker push 192.168.100.91:80/product-test:0.0.1接着再把消费者的jar也打好，名字我用的192.168.100.91:80/consumer-test:0.0.1
3、在kubernetes的default空间中创建一个用户，给以最高权限
master节点创建account.yaml
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: test #ClusterRoleBinding的名字
subjects:
- kind: ServiceAccount
name: test #serviceaccount资源对象的name
namespace: default #serviceaccount的namespace
roleRef:
kind: ClusterRole
name: cluster-admin #k8s集群中最高权限的角色
apiGroup: rbac.authorization.k8s.io
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: test # ServiceAccount的名字
namespace: default # serviceaccount的namespace
labels:
app: test #ServiceAccount的标签运行apply -f account.yaml
4、创建product-deployment.yaml
注意serviceAccountName要使用上面创建的test
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: product-test
spec:
replicas: 2
template:
metadata:
labels:
app: web
spec:
containers:
- name: product-test-instance
image: 192.168.100.91:80/product-test:0.0.1
ports:
- containerPort: 8080
serviceAccountName: testkubectl apply -f product-deployment.yaml5、创建product-service.yaml
注意这个serivice不使用Nodeport的方式向外发布
---
apiVersion: v1
kind: Service
metadata:
name: product-test-service
labels:
name: product-test-service
spec:
#type: NodePort #这里代表是NodePort类型的
ports:
- port: 8080 #这里的端口和clusterIP对应，即ip:8080,供内部访问。
targetPort: 8080 #端口一定要和container暴露出来的端口对应
protocol: TCP
# nodePort: 32143 # 所有的节点都会开放此端口，此端口供外部调用。
selector:
app: web #这里选择器一定要选择容器的标签，之前写name:kube-node是错的。kubectl apply -f product-service.yaml6、创建consumer-deployments.yaml
注意serviceAccountName要使用上面创建的test
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: consumer-test
spec:
replicas: 2
template:
metadata:
labels:
app: web-consumer
spec:
containers:
- name: consumer-test-instance
image: 192.168.100.91:80/consumer-test:0.0.1
ports:
- containerPort: 8080
serviceAccountName: testkubectl apply -f consumer-deployment.yaml7、创建consumer-service.yaml
注意这个serivice使用Nodeport的方式向外发布,测试访问
---
apiVersion: v1
kind: Service
metadata:
name: consumer-test-service
labels:
name: consumer-test-service
spec:
type: NodePort #这里代表是NodePort类型的
ports:
- port: 8080 #这里的端口和clusterIP对应，即ip:8080,供内部访问。
targetPort: 8080 #端口一定要和container暴露出来的端口对应
protocol: TCP
nodePort: 32144 # 所有的节点都会开放此端口，此端口供外部调用。
selector:
app: web-consumer #这里选择器一定要选择容器的标签，之前写name:kube-node是错的。kubectl apply -f consumer-service.yaml8、从浏览器访问http://[k8s一个Node的IP]:32144/properties
返回了“[“properties1”,“properties2”]”，证明调用成功了
三、spring cloud kubernetes调用过程
从网上找到调用过程图：
从上图可以看出product-infra-consumer在调用product-infra-service时，通过FeignClient组件拿到service name信息，最底层通过ok-http3，根据service name 调用 api server 获取该service下对应的Pod信息，拿到Pod信息后通过，轮询的方式向这些pod发送请求。spring-cloud-starter-kubernetes-ribbon组件中的KubernetesServerList 继承了 ribbon-loaderbanlancer组件中的AbstractServerList以及实现了 ServerList类中的方法，并通过 KubernetesClient提供的能力向k8s api server 发送请求信息。
