Guest User

Untitled

a guest
Oct 26th, 2017
98
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 16.08 KB | None | 0 0
  1. package kubernetes
  2.  
  3. import (
  4. "encoding/json"
  5. "fmt"
  6. "regexp"
  7. "strings"
  8.  
  9. "golang.org/x/net/context"
  10. "k8s.io/kubernetes/pkg/api"
  11. client "k8s.io/kubernetes/pkg/client/unversioned"
  12. "k8s.io/kubernetes/pkg/credentialprovider"
  13.  
  14. "gitlab.com/gitlab-org/gitlab-runner/common"
  15. "gitlab.com/gitlab-org/gitlab-runner/executors"
  16. )
  17.  
  18. var (
  19. executorOptions = executors.ExecutorOptions{
  20. SharedBuildsDir: false,
  21. Shell: common.ShellScriptInfo{
  22. Shell: "bash",
  23. Type: common.NormalShell,
  24. RunnerCommand: "/usr/bin/gitlab-runner-helper",
  25. },
  26. ShowHostname: true,
  27. }
  28. )
  29.  
  30. type kubernetesOptions struct {
  31. Image common.Image
  32. Services common.Services
  33. }
  34.  
  35. type executor struct {
  36. executors.AbstractExecutor
  37.  
  38. kubeClient *client.Client
  39. pod *api.Pod
  40. credentials *api.Secret
  41. options *kubernetesOptions
  42.  
  43. namespaceOverwrite string
  44. serviceAccountOverwrite string
  45. buildLimits api.ResourceList
  46. serviceLimits api.ResourceList
  47. helperLimits api.ResourceList
  48. buildRequests api.ResourceList
  49. serviceRequests api.ResourceList
  50. helperRequests api.ResourceList
  51. pullPolicy common.KubernetesPullPolicy
  52. }
  53.  
  54. func (s *executor) setupResources() error {
  55. var err error
  56.  
  57. // Limit
  58. if s.buildLimits, err = limits(s.Config.Kubernetes.CPULimit, s.Config.Kubernetes.MemoryLimit); err != nil {
  59. return fmt.Errorf("invalid build limits specified: %s", err.Error())
  60. }
  61.  
  62. if s.serviceLimits, err = limits(s.Config.Kubernetes.ServiceCPULimit, s.Config.Kubernetes.ServiceMemoryLimit); err != nil {
  63. return fmt.Errorf("invalid service limits specified: %s", err.Error())
  64. }
  65.  
  66. if s.helperLimits, err = limits(s.Config.Kubernetes.HelperCPULimit, s.Config.Kubernetes.HelperMemoryLimit); err != nil {
  67. return fmt.Errorf("invalid helper limits specified: %s", err.Error())
  68. }
  69.  
  70. // Requests
  71. if s.buildRequests, err = limits(s.Config.Kubernetes.CPURequest, s.Config.Kubernetes.MemoryRequest); err != nil {
  72. return fmt.Errorf("invalid build requests specified: %s", err.Error())
  73. }
  74.  
  75. if s.serviceRequests, err = limits(s.Config.Kubernetes.ServiceCPURequest, s.Config.Kubernetes.ServiceMemoryRequest); err != nil {
  76. return fmt.Errorf("invalid service requests specified: %s", err.Error())
  77. }
  78.  
  79. if s.helperRequests, err = limits(s.Config.Kubernetes.HelperCPURequest, s.Config.Kubernetes.HelperMemoryRequest); err != nil {
  80. return fmt.Errorf("invalid helper requests specified: %s", err.Error())
  81. }
  82. return nil
  83. }
  84.  
  85. func (s *executor) Prepare(options common.ExecutorPrepareOptions) (err error) {
  86. if err = s.AbstractExecutor.Prepare(options); err != nil {
  87. return err
  88. }
  89.  
  90. if s.BuildShell.PassFile {
  91. return fmt.Errorf("kubernetes doesn't support shells that require script file")
  92. }
  93.  
  94. if s.kubeClient, err = getKubeClient(options.Config.Kubernetes); err != nil {
  95. return fmt.Errorf("error connecting to Kubernetes: %s", err.Error())
  96. }
  97.  
  98. if err = s.setupResources(); err != nil {
  99. return err
  100. }
  101.  
  102. if s.pullPolicy, err = s.Config.Kubernetes.PullPolicy.Get(); err != nil {
  103. return err
  104. }
  105.  
  106. if err = s.overwriteNamespace(options.Build); err != nil {
  107. return err
  108. }
  109.  
  110. if err = s.overwriteServiceAccount(options.Build); err != nil {
  111. return err
  112. }
  113.  
  114. s.prepareOptions(options.Build)
  115.  
  116. if err = s.checkDefaults(); err != nil {
  117. return err
  118. }
  119.  
  120. s.Println("Using Kubernetes executor with image", s.options.Image.Name, "...")
  121.  
  122. return nil
  123. }
  124.  
  125. func (s *executor) Run(cmd common.ExecutorCommand) error {
  126. s.Debugln("Starting Kubernetes command...")
  127.  
  128. if s.pod == nil {
  129. err := s.setupCredentials()
  130. if err != nil {
  131. return err
  132. }
  133.  
  134. err = s.setupBuildPod()
  135. if err != nil {
  136. return err
  137. }
  138. }
  139.  
  140. containerName := "build"
  141. containerCommand := s.BuildShell.DockerCommand
  142. if cmd.Predefined {
  143. containerName = "helper"
  144. containerCommand = common.ContainerCommandBuild
  145. }
  146.  
  147. ctx, cancel := context.WithCancel(context.Background())
  148. defer cancel()
  149.  
  150. select {
  151. case err := <-s.runInContainer(ctx, containerName, containerCommand, cmd.Script):
  152. if err != nil && strings.Contains(err.Error(), "executing in Docker Container") {
  153. return &common.BuildError{Inner: err}
  154. }
  155. return err
  156.  
  157. case <-cmd.Context.Done():
  158. return fmt.Errorf("build aborted")
  159. }
  160. }
  161.  
  162. func (s *executor) Cleanup() {
  163. if s.pod != nil {
  164. err := s.kubeClient.Pods(s.pod.Namespace).Delete(s.pod.Name, nil)
  165. if err != nil {
  166. s.Errorln(fmt.Sprintf("Error cleaning up pod: %s", err.Error()))
  167. }
  168. }
  169. if s.credentials != nil {
  170. err := s.kubeClient.Secrets(s.pod.Namespace).Delete(s.credentials.Name)
  171. if err != nil {
  172. s.Errorln(fmt.Sprintf("Error cleaning up secrets: %s", err.Error()))
  173. }
  174. }
  175. closeKubeClient(s.kubeClient)
  176. s.AbstractExecutor.Cleanup()
  177. }
  178.  
  179. func (s *executor) buildContainer(name, image string, imageDefinition common.Image, requests, limits api.ResourceList, command ...string) api.Container {
  180. privileged := false
  181. if s.Config.Kubernetes != nil {
  182. privileged = s.Config.Kubernetes.Privileged
  183. }
  184.  
  185. if len(command) == 0 && len(imageDefinition.Command) > 0 {
  186. command = imageDefinition.Command
  187. }
  188.  
  189. var args []string
  190. if len(imageDefinition.Entrypoint) > 0 {
  191. args = command
  192. command = imageDefinition.Entrypoint
  193. }
  194.  
  195. return api.Container{
  196. Name: name,
  197. Image: image,
  198. ImagePullPolicy: api.PullPolicy(s.pullPolicy),
  199. Command: command,
  200. Args: args,
  201. Env: buildVariables(s.Build.GetAllVariables().PublicOrInternal()),
  202. Resources: api.ResourceRequirements{
  203. Limits: limits,
  204. Requests: requests,
  205. },
  206. VolumeMounts: s.getVolumeMounts(),
  207. SecurityContext: &api.SecurityContext{
  208. Privileged: &privileged,
  209. },
  210. Stdin: true,
  211. }
  212. }
  213.  
  214. func (s *executor) getVolumeMounts() (mounts []api.VolumeMount) {
  215. path := strings.Split(s.Build.BuildDir, "/")
  216. path = path[:len(path)-1]
  217.  
  218. mounts = append(mounts, api.VolumeMount{
  219. Name: "repo",
  220. MountPath: strings.Join(path, "/"),
  221. })
  222.  
  223. for _, mount := range s.Config.Kubernetes.Volumes.HostPaths {
  224. mounts = append(mounts, api.VolumeMount{
  225. Name: mount.Name,
  226. MountPath: mount.MountPath,
  227. ReadOnly: mount.ReadOnly,
  228. })
  229. }
  230.  
  231. for _, mount := range s.Config.Kubernetes.Volumes.Secrets {
  232. mounts = append(mounts, api.VolumeMount{
  233. Name: mount.Name,
  234. MountPath: mount.MountPath,
  235. ReadOnly: mount.ReadOnly,
  236. })
  237. }
  238.  
  239. for _, mount := range s.Config.Kubernetes.Volumes.PVCs {
  240. mounts = append(mounts, api.VolumeMount{
  241. Name: mount.Name,
  242. MountPath: mount.MountPath,
  243. ReadOnly: mount.ReadOnly,
  244. })
  245. }
  246.  
  247. for _, mount := range s.Config.Kubernetes.Volumes.ConfigMaps {
  248. mounts = append(mounts, api.VolumeMount{
  249. Name: mount.Name,
  250. MountPath: mount.MountPath,
  251. ReadOnly: mount.ReadOnly,
  252. })
  253. }
  254.  
  255. for _, mount := range s.Config.Kubernetes.Volumes.EmptyDirs {
  256. mounts = append(mounts, api.VolumeMount{
  257. Name: mount.Name,
  258. MountPath: mount.MountPath,
  259. })
  260. }
  261.  
  262. return
  263. }
  264.  
  265. func (s *executor) getVolumes() (volumes []api.Volume) {
  266. volumes = append(volumes, api.Volume{
  267. Name: "repo",
  268. VolumeSource: api.VolumeSource{
  269. EmptyDir: &api.EmptyDirVolumeSource{},
  270. },
  271. })
  272.  
  273. for _, volume := range s.Config.Kubernetes.Volumes.HostPaths {
  274. path := volume.HostPath
  275. // Make backward compatible with syntax introduced in version 9.3.0
  276. if path == "" {
  277. path = volume.MountPath
  278. }
  279.  
  280. volumes = append(volumes, api.Volume{
  281. Name: volume.Name,
  282. VolumeSource: api.VolumeSource{
  283. HostPath: &api.HostPathVolumeSource{
  284. Path: path,
  285. },
  286. },
  287. })
  288. }
  289.  
  290. for _, volume := range s.Config.Kubernetes.Volumes.Secrets {
  291. items := []api.KeyToPath{}
  292. for key, path := range volume.Items {
  293. items = append(items, api.KeyToPath{Key: key, Path: path})
  294. }
  295.  
  296. volumes = append(volumes, api.Volume{
  297. Name: volume.Name,
  298. VolumeSource: api.VolumeSource{
  299. Secret: &api.SecretVolumeSource{
  300. SecretName: volume.Name,
  301. Items: items,
  302. },
  303. },
  304. })
  305. }
  306.  
  307. for _, volume := range s.Config.Kubernetes.Volumes.PVCs {
  308. volumes = append(volumes, api.Volume{
  309. Name: volume.Name,
  310. VolumeSource: api.VolumeSource{
  311. PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{
  312. ClaimName: volume.Name,
  313. ReadOnly: volume.ReadOnly,
  314. },
  315. },
  316. })
  317. }
  318.  
  319. for _, volume := range s.Config.Kubernetes.Volumes.ConfigMaps {
  320. items := []api.KeyToPath{}
  321. for key, path := range volume.Items {
  322. items = append(items, api.KeyToPath{Key: key, Path: path})
  323. }
  324.  
  325. volumes = append(volumes, api.Volume{
  326. Name: volume.Name,
  327. VolumeSource: api.VolumeSource{
  328. ConfigMap: &api.ConfigMapVolumeSource{
  329. LocalObjectReference: api.LocalObjectReference{
  330. Name: volume.Name,
  331. },
  332. Items: items,
  333. },
  334. },
  335. })
  336. }
  337.  
  338. for _, volume := range s.Config.Kubernetes.Volumes.EmptyDirs {
  339. volumes = append(volumes, api.Volume{
  340. Name: volume.Name,
  341. VolumeSource: api.VolumeSource{
  342. EmptyDir: &api.EmptyDirVolumeSource{
  343. Medium: api.StorageMedium(volume.Medium),
  344. },
  345. },
  346. })
  347. }
  348.  
  349. return
  350. }
  351.  
  352. func (s *executor) setupCredentials() error {
  353. authConfigs := make(map[string]credentialprovider.DockerConfigEntry)
  354.  
  355. for _, credentials := range s.Build.Credentials {
  356. if credentials.Type != "registry" {
  357. continue
  358. }
  359.  
  360. authConfigs[credentials.URL] = credentialprovider.DockerConfigEntry{
  361. Username: credentials.Username,
  362. Password: credentials.Password,
  363. }
  364. }
  365.  
  366. if len(authConfigs) == 0 {
  367. return nil
  368. }
  369.  
  370. dockerCfgContent, err := json.Marshal(authConfigs)
  371. if err != nil {
  372. return err
  373. }
  374.  
  375. secret := api.Secret{}
  376. secret.GenerateName = s.Build.ProjectUniqueName()
  377. secret.Namespace = s.Config.Kubernetes.Namespace
  378. secret.Type = api.SecretTypeDockercfg
  379. secret.Data = map[string][]byte{}
  380. secret.Data[api.DockerConfigKey] = dockerCfgContent
  381.  
  382. s.credentials, err = s.kubeClient.Secrets(s.Config.Kubernetes.Namespace).Create(&secret)
  383. if err != nil {
  384. return err
  385. }
  386. return nil
  387. }
  388.  
  389. func (s *executor) setupBuildPod() error {
  390. services := make([]api.Container, len(s.options.Services))
  391. for i, service := range s.options.Services {
  392. resolvedImage := s.Build.GetAllVariables().ExpandValue(service.Name)
  393. services[i] = s.buildContainer(fmt.Sprintf("svc-%d", i), resolvedImage, service, s.serviceRequests, s.serviceLimits)
  394. }
  395. labels := make(map[string]string)
  396. for k, v := range s.Build.Runner.Kubernetes.PodLabels {
  397. labels[k] = s.Build.Variables.ExpandValue(v)
  398. }
  399.  
  400. var imagePullSecrets []api.LocalObjectReference
  401. for _, imagePullSecret := range s.Config.Kubernetes.ImagePullSecrets {
  402. imagePullSecrets = append(imagePullSecrets, api.LocalObjectReference{Name: imagePullSecret})
  403. }
  404.  
  405. if s.credentials != nil {
  406. imagePullSecrets = append(imagePullSecrets, api.LocalObjectReference{Name: s.credentials.Name})
  407. }
  408.  
  409. buildImage := s.Build.GetAllVariables().ExpandValue(s.options.Image.Name)
  410. pod, err := s.kubeClient.Pods(s.Config.Kubernetes.Namespace).Create(&api.Pod{
  411. ObjectMeta: api.ObjectMeta{
  412. GenerateName: s.Build.ProjectUniqueName(),
  413. Namespace: s.Config.Kubernetes.Namespace,
  414. Labels: labels,
  415. },
  416. Spec: api.PodSpec{
  417. Volumes: s.getVolumes(),
  418. ServiceAccountName: s.Config.Kubernetes.ServiceAccount,
  419. RestartPolicy: api.RestartPolicyNever,
  420. NodeSelector: s.Config.Kubernetes.NodeSelector,
  421. Containers: append([]api.Container{
  422. // TODO use the build and helper template here
  423. s.buildContainer("build", buildImage, s.options.Image, s.buildRequests, s.buildLimits, s.BuildShell.DockerCommand...),
  424. s.buildContainer("helper", s.Config.Kubernetes.GetHelperImage(), common.Image{}, s.helperRequests, s.helperLimits, s.BuildShell.DockerCommand...),
  425. }, services...),
  426. TerminationGracePeriodSeconds: &s.Config.Kubernetes.TerminationGracePeriodSeconds,
  427. ImagePullSecrets: imagePullSecrets,
  428. },
  429. })
  430.  
  431. if err != nil {
  432. return err
  433. }
  434.  
  435. s.pod = pod
  436.  
  437. return nil
  438. }
  439.  
  440. func (s *executor) runInContainer(ctx context.Context, name string, command []string, script string) <-chan error {
  441. errc := make(chan error, 1)
  442. go func() {
  443. defer close(errc)
  444.  
  445. status, err := waitForPodRunning(ctx, s.kubeClient, s.pod, s.Trace, s.Config.Kubernetes)
  446.  
  447. if err != nil {
  448. errc <- err
  449. return
  450. }
  451.  
  452. if status != api.PodRunning {
  453. errc <- fmt.Errorf("pod failed to enter running state: %s", status)
  454. return
  455. }
  456.  
  457. config, err := getKubeClientConfig(s.Config.Kubernetes)
  458.  
  459. if err != nil {
  460. errc <- err
  461. return
  462. }
  463.  
  464. exec := ExecOptions{
  465. PodName: s.pod.Name,
  466. Namespace: s.pod.Namespace,
  467. ContainerName: name,
  468. Command: command,
  469. In: strings.NewReader(script),
  470. Out: s.Trace,
  471. Err: s.Trace,
  472. Stdin: true,
  473. Config: config,
  474. Client: s.kubeClient,
  475. Executor: &DefaultRemoteExecutor{},
  476. }
  477.  
  478. errc <- exec.Run()
  479. }()
  480.  
  481. return errc
  482. }
  483.  
  484. func (s *executor) prepareOptions(job *common.Build) {
  485. s.options = &kubernetesOptions{}
  486. s.options.Image = job.Image
  487. for _, service := range job.Services {
  488. if service.Name == "" {
  489. continue
  490. }
  491. s.options.Services = append(s.options.Services, service)
  492. }
  493. }
  494.  
  495. // checkDefaults Defines the configuration for the Pod on Kubernetes
  496. func (s *executor) checkDefaults() error {
  497. if s.options.Image.Name == "" {
  498. if s.Config.Kubernetes.Image == "" {
  499. return fmt.Errorf("no image specified and no default set in config")
  500. }
  501.  
  502. s.options.Image = common.Image{
  503. Name: s.Config.Kubernetes.Image,
  504. }
  505. }
  506.  
  507. if s.Config.Kubernetes.Namespace == "" {
  508. s.Warningln("Namespace is empty, therefore assuming 'default'.")
  509. s.Config.Kubernetes.Namespace = "default"
  510. }
  511.  
  512. s.Println("Using Kubernetes namespace:", s.Config.Kubernetes.Namespace)
  513.  
  514. return nil
  515. }
  516.  
  517. // overwriteNamespace checks for variable in order to overwrite the configured
  518. // namespace, as long as it complies to validation regular-expression, when
  519. // expression is empty the overwrite is disabled.
  520. func (s *executor) overwriteNamespace(job *common.Build) error {
  521. if s.Config.Kubernetes.NamespaceOverwriteAllowed == "" {
  522. s.Debugln("Configuration entry 'namespace_overwrite_allowed' is empty, using configured namespace.")
  523. return nil
  524. }
  525.  
  526. // looking for namespace overwrite variable, and expanding for interpolation
  527. s.namespaceOverwrite = job.Variables.Expand().Get("KUBERNETES_NAMESPACE_OVERWRITE")
  528. if s.namespaceOverwrite == "" {
  529. return nil
  530. }
  531.  
  532. if err := overwriteRegexCheck(s.Config.Kubernetes.NamespaceOverwriteAllowed, s.namespaceOverwrite); err != nil {
  533. return err
  534. }
  535.  
  536. s.Println("Overwritting configured namespace, from", s.Config.Kubernetes.Namespace, "to", s.namespaceOverwrite)
  537. s.Config.Kubernetes.Namespace = s.namespaceOverwrite
  538.  
  539. return nil
  540. }
  541.  
  542. // overwriteSercviceAccount checks for variable in order to overwrite the configured
  543. // service account, as long as it complies to validation regular-expression, when
  544. // expression is empty the overwrite is disabled.
  545. func (s *executor) overwriteServiceAccount(job *common.Build) error {
  546. if s.Config.Kubernetes.ServiceAccountOverwriteAllowed == "" {
  547. s.Debugln("Configuration entry 'service_account_overwrite_allowed' is empty, disabling override.")
  548. return nil
  549. }
  550.  
  551. s.serviceAccountOverwrite = job.Variables.Expand().Get("KUBERNETES_SERVICE_ACCOUNT_OVERWRITE")
  552. if s.serviceAccountOverwrite == "" {
  553. return nil
  554. }
  555.  
  556. if err := overwriteRegexCheck(s.Config.Kubernetes.ServiceAccountOverwriteAllowed, s.serviceAccountOverwrite); err != nil {
  557. return err
  558. }
  559.  
  560. s.Println("Overwritting configured ServiceAccount, from", s.Config.Kubernetes.ServiceAccount, "to", s.serviceAccountOverwrite)
  561. s.Config.Kubernetes.ServiceAccount = s.serviceAccountOverwrite
  562.  
  563. return nil
  564. }
  565.  
  566. //overwriteRegexCheck check if the regex provided for overwriting a config field matches the
  567. //paramether provided, returns error if doesn't match
  568. func overwriteRegexCheck(regex, value string) error {
  569. var err error
  570. var r *regexp.Regexp
  571. if r, err = regexp.Compile(regex); err != nil {
  572. return err
  573. }
  574.  
  575. if match := r.MatchString(value); !match {
  576. return fmt.Errorf("Provided value %s does not match regex %s", value, regex)
  577. }
  578. return nil
  579. }
  580.  
  581. func createFn() common.Executor {
  582. return &executor{
  583. AbstractExecutor: executors.AbstractExecutor{
  584. ExecutorOptions: executorOptions,
  585. },
  586. }
  587. }
  588.  
  589. func featuresFn(features *common.FeaturesInfo) {
  590. features.Variables = true
  591. features.Image = true
  592. features.Services = true
  593. features.Artifacts = true
  594. features.Cache = true
  595. }
  596.  
  597. func init() {
  598. common.RegisterExecutor("kubernetes", executors.DefaultExecutorProvider{
  599. Creator: createFn,
  600. FeaturesUpdater: featuresFn,
  601. })
  602. }
Add Comment
Please, Sign In to add comment