Advertisement
Guest User

Untitled

a guest
Feb 25th, 2016
70
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 10.09 KB | None | 0 0
  1. /*
  2. This is a post checking tool that performs role specific tests on the roles to ensure that everything
  3. is installed and running correctly
  4. */
  5.  
  6. package postcheck
  7.  
  8. import (
  9. "errors"
  10. "flag"
  11. "fmt"
  12. log "github.com/Sirupsen/logrus"
  13. "github.com/jeffail/gabs"
  14. "golang.org/x/crypto/ssh"
  15. "gopkg.in/mgo.v2"
  16. "gopkg.in/mgo.v2/bson"
  17. "lib/check"
  18. "lib/sshutility"
  19. "strings"
  20. "time"
  21. "utils/helper"
  22. )
  23.  
  24. var (
  25. hostIps = ""
  26. username = ""
  27. configJson *gabs.Container
  28. password = ""
  29. hostsToCheck []string
  30. postCheckErr = errors.New("Postcheck failed!")
  31. )
  32.  
  33. type HostsMappingDoc struct {
  34. Cluster string
  35. Mapping map[string]interface{}
  36. }
  37.  
  38. func init() {
  39. flag.StringVar(&hostIps, "mongo_hosts", "", "mongodb IP's")
  40. flag.StringVar(&username, "username", "", "mongodb username")
  41. flag.StringVar(&password, "password", "", "mongodb password")
  42. }
  43.  
  44. // Method to check if the mongodb details are not empty
  45. func checkInfo(hostIps string, username string, password string) error {
  46. if hostIps == "" || username == "" || password == "" {
  47. return errors.New("Please provide the mongodb IP/s, mongodb username and password")
  48. }
  49. return nil
  50. }
  51.  
  52. func Exec(configJsonDeploy *gabs.Container) error {
  53.  
  54. flag.Parse()
  55.  
  56. // Check if the input config is used
  57. if configJsonDeploy != nil {
  58. // Collect mongodb information and retrieve the config form the database
  59. mongoUsername, _ := configJsonDeploy.Path("user_input.mongodb.username").Data().(string)
  60. mongoPassword, _ := configJsonDeploy.Path("user_input.mongodb.password").Data().(string)
  61. var mongoIps []string
  62. helper.ExtractStringArray(&mongoIps, configJsonDeploy, "user_input.mongodb.hosts")
  63. var IPstring string
  64. // Check if a replica set is used
  65. if len(mongoIps) == 3 {
  66. IPstring = mongoIps[0] + "," + mongoIps[1] + "," + mongoIps[2]
  67. } else {
  68. IPstring = mongoIps[0]
  69. }
  70. configJson = getConfigJsonFromDatabase(IPstring, mongoUsername, mongoPassword)
  71.  
  72. } else {
  73. // Check if mongodb details are provided
  74. err := checkInfo(hostIps, username, password)
  75. if err != nil {
  76. log.Fatal(err)
  77. }
  78. // Get the config from the mongo database
  79. configJson = getConfigJsonFromDatabase(hostIps, username, password)
  80. }
  81. //fmt.Println(configJson)
  82. log.Info("Running checks...")
  83.  
  84. // Get in the username and password for ssh
  85. sshUsername := check.GetUser(configJson)
  86. sshPassword, _ := configJson.Path("user_input.ssh.password").Data().(string)
  87. sshPem, okSshPem := configJson.Path("user_input.ssh.pem").Data().(string)
  88. sshPort := configJson.Path("user_input.ssh.port").Data()
  89. sshPortStr := fmt.Sprintf("%v", sshPort)
  90. sshConn := sshutility.SSHConnection{
  91. UserName: sshUsername,
  92. PassWord: sshPassword,
  93. PemFileName: sshPem,
  94. UsePem: okSshPem,
  95. DefaultPort: sshPortStr,
  96. }
  97.  
  98. hostTypes := []string{"arrow_cloud", "mongodb"}
  99. // Check if the arrowdb cluster is present
  100. arrow_dbCnts, err := configJson.Path("user_input.arrow_db").Children()
  101. if err == nil && arrow_dbCnts != nil {
  102. hostTypes = append(hostTypes, "arrow_db")
  103. }
  104. var component_ips map[string][]string = check.GetMap(configJson, hostTypes)
  105. var channelArray []<-chan string
  106. // Iterate over the go map collecting key's and value's to write the hosts to a file
  107. for prod_component, ips := range component_ips {
  108.  
  109. for x := 0; x < len(ips); x++ {
  110. ip := ips[x]
  111. // Run the checkHost method in parallel
  112. result := connectHost2(sshConn, configJson, prod_component, ip)
  113. channelArray = append(channelArray, result)
  114. }
  115. }
  116. var passed = true
  117.  
  118. fmt.Println(len(channelArray))
  119. for x := 0; x < len(channelArray); x++ {
  120. select {
  121. case chanResult := <-channelArray[x]:
  122. boolResult := strings.Split(chanResult, "--")
  123. result := boolResult[1]
  124. if result == "failed" {
  125. passed = false
  126. }
  127. }
  128. }
  129.  
  130. fmt.Println("passed = ", passed)
  131.  
  132. // Final check to see is the postcheck passed, otherwise return an error
  133. if passed == true {
  134. log.Info("Arrowcloud_postcheck complete")
  135. return nil
  136. } else {
  137. return postCheckErr
  138. }
  139.  
  140. }
  141.  
  142. // SSH and run the postCheck method on the current node
  143. func connectHost2(sshConn sshutility.SSHConnection, configJson *gabs.Container, prod_component string, hostIP string) <-chan string {
  144. c := make(chan string)
  145. go connectHost(sshConn, configJson, c, prod_component, hostIP)
  146. return c
  147.  
  148. }
  149.  
  150. // SSH and run the postCheck method on the current node
  151. func connectHost(sshConn sshutility.SSHConnection, configJson *gabs.Container, c chan string, prod_component string, hostIP string) {
  152.  
  153. node := map[string]string{
  154. "host": hostIP,
  155. }
  156.  
  157. // Point towards the client in the ssh library
  158. sshClient, err := sshConn.Connect(node)
  159.  
  160. if err != nil {
  161. log.Fatalf("Error when connecting to node[%s(%s)]: %v", prod_component, hostIP, err)
  162. }
  163. defer sshClient.Close()
  164. passed := postCheck(sshClient, configJson, prod_component, hostIP)
  165. var name = prod_component + "." + hostIP
  166. var result = ""
  167. if passed == true {
  168. result = name + "--passed"
  169. } else {
  170. result = name + "--failed"
  171. }
  172.  
  173. c <- result
  174.  
  175. }
  176.  
  177. // Method that runs role specific checks to ensure certain tasks/services are running
  178. func postCheck(sshClient *ssh.Client, configJson *gabs.Container, prod_component string, hostIP string) bool {
  179.  
  180. var passed = true
  181.  
  182. // Split up the component names into cluster and role
  183. prodCompArray := strings.Split(prod_component, ".")
  184. product, componentName := prodCompArray[0], prodCompArray[1]
  185.  
  186. // Get the arrowdb_url_prefix and domain_name form the user_input
  187. arrowdb_url_prefix, _ := configJson.Path("user_input.arrowdb_url_prefix").Data().(string)
  188. domain_name, _ := configJson.Path("user_input.domain_name").Data().(string)
  189.  
  190. // Switch cases to distinguish between the roles
  191. switch product {
  192. case "arrow_cloud":
  193. switch componentName {
  194. case "stratus":
  195.  
  196. curl := "curl " + hostIP + ":7000/ping.json"
  197. var expected = "\"ping\": \"ok\""
  198. passed = runCurl(sshClient, curl, product, componentName, hostIP, expected)
  199.  
  200. case "log_aggregator":
  201.  
  202. case "app_haproxy":
  203.  
  204. case "admin_haproxy":
  205.  
  206. case "admin":
  207.  
  208. case "docker_registry":
  209.  
  210. case "container_node":
  211.  
  212. curlSession, _ := sshClient.NewSession()
  213. curl := "curl " + hostIP + ":9090/ping"
  214. if err := curlSession.Run(curl); err != nil {
  215. passed = false
  216. log.Error("cannot '"+curl+"' from <"+product+"."+componentName+">, IP : ", hostIP)
  217. }
  218. defer curlSession.Close()
  219.  
  220. default:
  221. log.Error("the hosts are unreadable")
  222. return false
  223. }
  224. case "arrow_db":
  225. switch componentName {
  226. case "api":
  227.  
  228. curlSession, _ := sshClient.NewSession()
  229. curl := "curl " + hostIP + ":8082"
  230. if err := curlSession.Run(curl); err != nil {
  231. passed = false
  232. log.Error("cannot '"+curl+"' from <"+product+"."+componentName+">, IP : ", hostIP)
  233. } else {
  234. defer curlSession.Close()
  235. }
  236.  
  237. case "upload":
  238.  
  239. delayed_jobSession, _ := sshClient.NewSession()
  240. output, _ := delayed_jobSession.Output("ps -ef")
  241. outputString := string(output[:])
  242. defer delayed_jobSession.Close()
  243. if !strings.Contains(outputString, "delayed_job") {
  244. log.Error("delayed_job is not running on " + hostIP + " \n")
  245. passed = false
  246. }
  247.  
  248. curlSession, _ := sshClient.NewSession()
  249. curl := "curl " + hostIP + ":8082"
  250. if err := curlSession.Run(curl); err != nil {
  251. passed = false
  252. log.Error("cannot '"+curl+"' from <"+product+"."+componentName+">, IP : ", hostIP)
  253. } else {
  254. defer curlSession.Close()
  255. }
  256.  
  257. case "admin":
  258.  
  259. case "haproxy":
  260.  
  261. curl := "curl --header \"Host: " + arrowdb_url_prefix + "." + domain_name + "\" " + hostIP + "/v1/admins/ping.json"
  262. var expected = "\"code\":200"
  263. passed = runCurl(sshClient, curl, product, componentName, hostIP, expected)
  264.  
  265. case "push":
  266.  
  267. default:
  268. log.Println("the hosts are unreadable")
  269. return false
  270. }
  271. }
  272. return passed
  273. }
  274.  
  275. // Method to cmpare the output of the test with the expected result
  276. func compareOutput(result string, curl string, product string, componentName string, hostIP string, expected string) bool {
  277. // Check if the output contains the result
  278. if !strings.Contains(result, expected) {
  279. return false
  280. }
  281. return true
  282. }
  283.  
  284. // Method that ssh's into the host and runs a curl command
  285. func runCurl(sshClient *ssh.Client, curl string, product string, componentName string, hostIP string, expected string) bool {
  286. result := ""
  287. // Check if the ssh.Client is not nil
  288. if sshClient != nil {
  289. curlSession, _ := sshClient.NewSession()
  290. out, _ := curlSession.Output(curl)
  291. result = string(out[:])
  292. // Make sure the output is not empty
  293. if result == "" {
  294. log.Error("cannot curl : " + curl)
  295. } else {
  296. defer curlSession.Close()
  297. }
  298. }
  299.  
  300. // Compare the output with the expected result
  301. var output = compareOutput(result, curl, product, componentName, hostIP, expected)
  302. if output == false {
  303. log.Error("cannot '"+curl+"' from <"+product+"."+componentName+">, IP : ", hostIP)
  304. return false
  305. }
  306. return true
  307. }
  308.  
  309. /**
  310. If user specifies hosts array in user_input only (does not assign hosts to arrowcloud/DB roles) we should get
  311. configuration from database which was saved when setting up the cluster.
  312. */
  313. func getConfigJsonFromDatabase(hostIps string, dbUsername string, dbPassword string) *gabs.Container {
  314.  
  315. mongoHosts := strings.Split(hostIps, ",")
  316.  
  317. // Append the port to each IP
  318. var mongoHostPorts []string
  319. for x := 0; x < len(mongoHosts); x++ {
  320. mongoHostPorts = append(mongoHostPorts, mongoHosts[x]+":27017")
  321. }
  322.  
  323. log.Infof("Retrieving configuration from database: %v", mongoHostPorts)
  324.  
  325. // Initialize info using the username, password and Host
  326. mongoDBDialInfo := &mgo.DialInfo{
  327. Addrs: mongoHostPorts,
  328. Timeout: 10 * time.Second,
  329. Database: "deployment_stacks",
  330. Username: dbUsername,
  331. Password: dbPassword,
  332. }
  333.  
  334. // Dial into mongo db using the replica set
  335. session, err := mgo.DialWithInfo(mongoDBDialInfo)
  336.  
  337. if err != nil {
  338. log.Fatalf("Failed to connect to mongo nodes: %s\n", err)
  339. }
  340. session.SetMode(mgo.Monotonic, true)
  341.  
  342. // Select the host_mapping collection
  343. c := session.DB("deployment_stacks").C("hosts_mapping")
  344.  
  345. result := HostsMappingDoc{}
  346.  
  347. // Find the record
  348. err = c.Find(bson.M{}).One(&result)
  349. if err != nil {
  350. log.Fatalf("Failed to get configuration from database. %v", err)
  351. }
  352.  
  353. // Create a json object for the host map
  354. configInMongo, err := gabs.Consume(result.Mapping)
  355. if err != nil {
  356. log.Fatalf("Failed to parsing configuration got from database. %v", err)
  357. }
  358.  
  359. return configInMongo
  360. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement