Advertisement
Guest User

Untitled

a guest
Feb 26th, 2016
107
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 11.02 KB | None | 0 0
  1. /*
  2. This is a post checking tool that performs role specific tests on the roles to ensure that everything
  3. is installed and running correctly
  4. */
  5.  
  6. package postcheck
  7.  
  8. import (
  9. "errors"
  10. "flag"
  11. "fmt"
  12. log "github.com/Sirupsen/logrus"
  13. "github.com/jeffail/gabs"
  14. "golang.org/x/crypto/ssh"
  15. "gopkg.in/mgo.v2"
  16. "gopkg.in/mgo.v2/bson"
  17. "lib/check"
  18. "lib/sshutility"
  19. "strings"
  20. "time"
  21. "utils/helper"
  22. )
  23.  
  24. var (
  25. hostIps = ""
  26. username = ""
  27. configJson *gabs.Container
  28. password = ""
  29. hostsToCheck []string
  30. postCheckErr error
  31. )
  32.  
  33. type HostsMappingDoc struct {
  34. Cluster string
  35. Mapping map[string]interface{}
  36. }
  37.  
  38. func init() {
  39. flag.StringVar(&hostIps, "mongo_hosts", "", "mongodb IP's")
  40. flag.StringVar(&username, "username", "", "mongodb username")
  41. flag.StringVar(&password, "password", "", "mongodb password")
  42. }
  43.  
  44. // Method to check if the mongodb details are not empty
  45. func checkInfo(hostIps string, username string, password string) error {
  46. if hostIps == "" || username == "" || password == "" {
  47. return errors.New("Please provide the mongodb IP/s, mongodb username and password")
  48. }
  49. return nil
  50. }
  51.  
  52. func Exec(configJsonDeploy *gabs.Container) {
  53.  
  54. flag.Parse()
  55.  
  56. // Check if the input config is used
  57. if configJsonDeploy != nil {
  58. // Collect mongodb information and retrieve the config form the database
  59. mongoUsername, _ := configJsonDeploy.Path("user_input.mongodb.username").Data().(string)
  60. mongoPassword, _ := configJsonDeploy.Path("user_input.mongodb.password").Data().(string)
  61. var mongoIps []string
  62. helper.ExtractStringArray(&mongoIps, configJsonDeploy, "user_input.mongodb.hosts")
  63. var IPstring string
  64. // Check if a replica set is used
  65. if len(mongoIps) == 3 {
  66. IPstring = mongoIps[0] + "," + mongoIps[1] + "," + mongoIps[2]
  67. } else {
  68. IPstring = mongoIps[0]
  69. }
  70. configJson = getConfigJsonFromDatabase(IPstring, mongoUsername, mongoPassword)
  71.  
  72. } else {
  73. // Check if mongodb details are provided
  74. err := checkInfo(hostIps, username, password)
  75. if err != nil {
  76. log.Fatal(err)
  77. }
  78. // Get the config from the mongo database
  79. configJson = getConfigJsonFromDatabase(hostIps, username, password)
  80. }
  81. //fmt.Println(configJson)
  82. log.Info("Running checks...")
  83.  
  84. //fmt.Println(configJson)
  85.  
  86. // Get in the username and password for ssh
  87. sshUsername := check.GetUser(configJson)
  88. sshPassword, _ := configJson.Path("user_input.ssh.password").Data().(string)
  89. sshPem, okSshPem := configJson.Path("user_input.ssh.pem").Data().(string)
  90. sshPort := configJson.Path("user_input.ssh.port").Data()
  91. sshPortStr := fmt.Sprintf("%v", sshPort)
  92. sshConn := sshutility.SSHConnection{
  93. UserName: sshUsername,
  94. PassWord: sshPassword,
  95. PemFileName: sshPem,
  96. UsePem: okSshPem,
  97. DefaultPort: sshPortStr,
  98. }
  99.  
  100. hostTypes := []string{"arrow_cloud"}
  101. // Check if the arrowdb cluster is present
  102. arrow_dbCnts, err := configJson.Path("user_input.arrow_db").Children()
  103. if err == nil && arrow_dbCnts != nil {
  104. hostTypes = append(hostTypes, "arrow_db")
  105. }
  106. var component_ips map[string][]string = check.GetMap(configJson, hostTypes)
  107. var channelArray []<-chan string
  108. //var Duplicate []string
  109. // Iterate over the go map collecting key's and value's to write the hosts to a file
  110.  
  111. var doneIPS []string
  112. for _, ips := range component_ips {
  113.  
  114. for x := 0; x < len(ips); x++ {
  115.  
  116. if !stringInSlice(ips[x], doneIPS) {
  117. //Duplicate = append(Duplicate, prod_component2)
  118.  
  119. var Duplicate []string
  120.  
  121. for prod_component2, ips2 := range component_ips {
  122.  
  123. for y := 0; y < len(ips2); y++ {
  124. ip2 := ips2[y]
  125. if ip2 == ips[x] {
  126. Duplicate = append(Duplicate, prod_component2)
  127. }
  128.  
  129. // if stringInSlice(ip2, Duplicate) {
  130. // Duplicate = append(Duplicate, prod_component2)
  131. // }
  132.  
  133. }
  134. }
  135. doneIPS = append(doneIPS, ips[x])
  136.  
  137. //log.Error("duplicated are ", Duplicate, " for the ip ", ips[x])
  138. // Run the checkHost method in parallel
  139. result := connectHost2(sshConn, configJson, Duplicate, ips[x])
  140. channelArray = append(channelArray, result)
  141. }
  142.  
  143. }
  144.  
  145. }
  146.  
  147. //log.Fatal("done")
  148.  
  149. var passed = true
  150. for x := 0; x < len(channelArray); x++ {
  151. //select {
  152. chanResult := <-channelArray[x]
  153. boolResult := strings.Split(chanResult, "--")
  154. result := boolResult[1]
  155. fmt.Println("result for "+boolResult[0]+" is ", result)
  156. if result == "failed" && postCheckErr == nil {
  157. postCheckErr = errors.New("Postcheck failed!")
  158. }
  159. }
  160.  
  161. fmt.Println("passed = ", passed)
  162.  
  163. if postCheckErr != nil {
  164. fmt.Println("postcheck failed")
  165. } else {
  166. fmt.Println("postcheck passed")
  167. }
  168.  
  169. }
  170.  
  171. // SSH and run the postCheck method on the current node
  172. func connectHost2(sshConn sshutility.SSHConnection, configJson *gabs.Container, prod_components []string, hostIP string) <-chan string {
  173. c := make(chan string)
  174. go connectHost(sshConn, configJson, c, prod_components, hostIP)
  175. //fmt.Println("returning channel : ", <-c)
  176. return c
  177.  
  178. }
  179.  
  180. // SSH and run the postCheck method on the current node
  181. func connectHost(sshConn sshutility.SSHConnection, configJson *gabs.Container, c chan string, prod_components []string, hostIP string) {
  182.  
  183. node := map[string]string{
  184. "host": hostIP,
  185. }
  186.  
  187. // Point towards the client in the ssh library
  188. sshClient, err := sshConn.Connect(node)
  189.  
  190. if err != nil {
  191. log.Fatalf("Error when connecting to node/s[%s(%s)]: %v", prod_components, hostIP, err)
  192. }
  193. var finalPass = true
  194. for x := 0; x < len(prod_components); x++ {
  195. passed := postCheck(sshClient, configJson, prod_components[x], hostIP)
  196. //var name = prod_component + "." + hostIP
  197. //var result = ""
  198. if passed != true {
  199. finalPass = false
  200. }
  201. }
  202.  
  203. defer sshClient.Close()
  204. if finalPass == true {
  205. c <- hostIP + "--passed"
  206. } else {
  207. c <- hostIP + "--failed"
  208. }
  209.  
  210. }
  211.  
  212. // Method that runs role specific checks to ensure certain tasks/services are running
  213. func postCheck(sshClient *ssh.Client, configJson *gabs.Container, prod_component string, hostIP string) bool {
  214.  
  215. var passed = true
  216.  
  217. // Split up the component names into cluster and role
  218. prodCompArray := strings.Split(prod_component, ".")
  219. product, componentName := prodCompArray[0], prodCompArray[1]
  220.  
  221. // Get the arrowdb_url_prefix and domain_name form the user_input
  222. arrowdb_url_prefix, _ := configJson.Path("user_input.arrowdb_url_prefix").Data().(string)
  223. domain_name, _ := configJson.Path("user_input.domain_name").Data().(string)
  224.  
  225. // Switch cases to distinguish between the roles
  226. switch product {
  227. case "arrow_cloud":
  228. switch componentName {
  229. case "stratus":
  230.  
  231. curl := "curl " + hostIP + ":7000/ping.json"
  232. var expected = "\"ping\": \"ok\""
  233. passed = runCurl(sshClient, curl, product, componentName, hostIP, expected)
  234.  
  235. case "log_aggregator":
  236.  
  237. case "app_haproxy":
  238.  
  239. case "admin_haproxy":
  240.  
  241. case "admin":
  242.  
  243. case "docker_registry":
  244.  
  245. case "container_node":
  246.  
  247. curlSession, _ := sshClient.NewSession()
  248. curl := "curl " + hostIP + ":9090/ping"
  249. if err := curlSession.Run(curl); err != nil {
  250. passed = false
  251. log.Error("cannot '"+curl+"' from <"+product+"."+componentName+">, IP : ", hostIP)
  252. } else {
  253. defer curlSession.Close()
  254. }
  255.  
  256. default:
  257. log.Error("the hosts are unreadable")
  258. return false
  259. }
  260. case "arrow_db":
  261. switch componentName {
  262. case "api":
  263.  
  264. curlSession, _ := sshClient.NewSession()
  265. curl := "curl " + hostIP + ":8082"
  266. if err := curlSession.Run(curl); err != nil {
  267. passed = false
  268. log.Error("cannot '"+curl+"' from <"+product+"."+componentName+">, IP : ", hostIP)
  269. } else {
  270. defer curlSession.Close()
  271. }
  272.  
  273. case "upload":
  274.  
  275. delayed_jobSession, _ := sshClient.NewSession()
  276. output, _ := delayed_jobSession.Output("ps -ef")
  277. outputString := string(output[:])
  278. defer delayed_jobSession.Close()
  279. if !strings.Contains(outputString, "delayed_job") {
  280. log.Error("delayed_job is not running on " + hostIP + " \n")
  281. passed = false
  282. }
  283.  
  284. curlSession, _ := sshClient.NewSession()
  285. curl := "curl " + hostIP + ":8082"
  286. if err := curlSession.Run(curl); err != nil {
  287. passed = false
  288. log.Error("cannot '"+curl+"' from <"+product+"."+componentName+">, IP : ", hostIP)
  289. } else {
  290. defer curlSession.Close()
  291. }
  292.  
  293. case "admin":
  294.  
  295. case "haproxy":
  296.  
  297. curl := "curl --header \"Host: " + arrowdb_url_prefix + "." + domain_name + "\" " + hostIP + "/v1/admins/ping.json"
  298. var expected = "\"code\":200"
  299. passed = runCurl(sshClient, curl, product, componentName, hostIP, expected)
  300.  
  301. case "push":
  302.  
  303. default:
  304. log.Println("the hosts are unreadable")
  305. return false
  306. }
  307. }
  308. return passed
  309. }
  310.  
  311. // Method to cmpare the output of the test with the expected result
  312. func compareOutput(result string, curl string, product string, componentName string, hostIP string, expected string) bool {
  313. // Check if the output contains the result
  314. if !strings.Contains(result, expected) {
  315. return false
  316. }
  317. return true
  318. }
  319.  
  320. // Method that ssh's into the host and runs a curl command
  321. func runCurl(sshClient *ssh.Client, curl string, product string, componentName string, hostIP string, expected string) bool {
  322. result := ""
  323. // Check if the ssh.Client is not nil
  324. if sshClient != nil {
  325. curlSession, _ := sshClient.NewSession()
  326. out, _ := curlSession.Output(curl)
  327. result = string(out[:])
  328. // Make sure the output is not empty
  329. if result == "" {
  330. log.Error("cannot curl : " + curl)
  331. } else {
  332. defer curlSession.Close()
  333. }
  334. }
  335.  
  336. // Compare the output with the expected result
  337. var output = compareOutput(result, curl, product, componentName, hostIP, expected)
  338. if output == false {
  339. log.Error("cannot '"+curl+"' from <"+product+"."+componentName+">, IP : ", hostIP)
  340. return false
  341. }
  342. return true
  343. }
  344.  
  345. /**
  346. If user specifies hosts array in user_input only (does not assign hosts to arrowcloud/DB roles) we should get
  347. configuration from database which was saved when setting up the cluster.
  348. */
  349. func getConfigJsonFromDatabase(hostIps string, dbUsername string, dbPassword string) *gabs.Container {
  350.  
  351. mongoHosts := strings.Split(hostIps, ",")
  352.  
  353. // Append the port to each IP
  354. var mongoHostPorts []string
  355. for x := 0; x < len(mongoHosts); x++ {
  356. mongoHostPorts = append(mongoHostPorts, mongoHosts[x]+":27017")
  357. }
  358.  
  359. log.Infof("Retrieving configuration from database: %v", mongoHostPorts)
  360.  
  361. // Initialize info using the username, password and Host
  362. mongoDBDialInfo := &mgo.DialInfo{
  363. Addrs: mongoHostPorts,
  364. Timeout: 10 * time.Second,
  365. Database: "deployment_stacks",
  366. Username: dbUsername,
  367. Password: dbPassword,
  368. }
  369.  
  370. // Dial into mongo db using the replica set
  371. session, err := mgo.DialWithInfo(mongoDBDialInfo)
  372.  
  373. if err != nil {
  374. log.Fatalf("Failed to connect to mongo nodes: %s\n", err)
  375. }
  376. session.SetMode(mgo.Monotonic, true)
  377.  
  378. // Select the host_mapping collection
  379. c := session.DB("deployment_stacks").C("hosts_mapping")
  380.  
  381. result := HostsMappingDoc{}
  382.  
  383. // Find the record
  384. err = c.Find(bson.M{}).One(&result)
  385. if err != nil {
  386. log.Fatalf("Failed to get configuration from database. %v", err)
  387. }
  388.  
  389. // Create a json object for the host map
  390. configInMongo, err := gabs.Consume(result.Mapping)
  391. if err != nil {
  392. log.Fatalf("Failed to parsing configuration got from database. %v", err)
  393. }
  394.  
  395. return configInMongo
  396. }
  397.  
  398. // Method to check if a string contain's a specified string
  399. func stringInSlice(a string, list []string) bool {
  400. for _, b := range list {
  401. if b == a {
  402. return true
  403. }
  404. }
  405. return false
  406. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement