Advertisement
Guest User

Untitled

a guest
Feb 5th, 2016
113
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 9.29 KB | None | 0 0
  1. /*
  2. This is a post checking tool that performs role specific tests on the roles to ensure that everything
  3. is installed and running correctly
  4. */
  5.  
  6. package main
  7.  
  8. import (
  9. "flag"
  10. "fmt"
  11. log "github.com/Sirupsen/logrus"
  12. "github.com/jeffail/gabs"
  13. "golang.org/x/crypto/ssh"
  14. "gopkg.in/mgo.v2"
  15. "gopkg.in/mgo.v2/bson"
  16. "lib/check"
  17. "lib/log_utility"
  18. "lib/sshutility"
  19. "lib/varfile"
  20. "strings"
  21. "time"
  22. "utils/helper"
  23. )
  24.  
  25. const COLLECTING_DIR = "output/cluster"
  26.  
  27. var (
  28. configFile = ""
  29. hostsToCheck []string
  30. )
  31.  
  32. type sshChan struct {
  33. name string
  34. }
  35.  
  36. type HostsMappingDoc struct {
  37. Cluster string
  38. Mapping map[string]interface{}
  39. }
  40.  
  41. func init() {
  42. flag.StringVar(&configFile, "config-file", "", "the deploy config file")
  43.  
  44. f, err := log_utility.CreateLogFile("collect")
  45. if err != nil {
  46. log.Fatalf("Error creating log file: %v", err)
  47. }
  48. log.SetOutput(f)
  49. log.AddHook(log_utility.NewUserFacingLogWithSpinnerHook())
  50. log.SetLevel(log.DebugLevel)
  51. }
  52.  
  53. func main() {
  54.  
  55. flag.Parse()
  56. if configFile == "" {
  57. log.Fatal("Missing parameter -config-file")
  58. }
  59.  
  60. log.Info("Start to collect arrowcloud debug information...\n")
  61.  
  62. configJson, err := varfile.GenerateVarfile(configFile)
  63. if err != nil {
  64. log.Fatalf("%s. Exiting...", err.Error())
  65. }
  66.  
  67. // Check if the hosts section exists
  68. hostCtns, err := configJson.Path("user_input.hosts").Children()
  69. if err == nil && hostCtns != nil {
  70. // If arrow_cloud and arrow_db are not specified (user did not define hosts-roles mapping, instead only hosts array
  71. // was specified.) get configuration in database as configJson
  72. configJson = getConfigJsonFromDatabase(configJson)
  73. } else {
  74. log.Debugf("hosts field does not exist or unreadable. err: %v", err)
  75. }
  76.  
  77. // Get in the username and password for ssh
  78. sshUsername := check.GetUser(configJson)
  79. sshPassword, _ := configJson.Path("user_input.ssh.password").Data().(string)
  80. sshPem, okSshPem := configJson.Path("user_input.ssh.pem").Data().(string)
  81. sshPort := configJson.Path("user_input.ssh.port").Data()
  82. sshPortStr := fmt.Sprintf("%v", sshPort)
  83. sshConn := sshutility.SSHConnection{
  84. UserName: sshUsername,
  85. PassWord: sshPassword,
  86. PemFileName: sshPem,
  87. UsePem: okSshPem,
  88. DefaultPort: sshPortStr,
  89. }
  90.  
  91. // Create a channel for parallel execution
  92. ch := make(chan sshChan)
  93.  
  94. numChannels := 0
  95.  
  96. hostTypes := []string{"arrow_cloud", "mongodb"}
  97. // Check if the arrowdb cluster is present
  98. arrow_dbCnts, err := configJson.Path("user_input.arrow_db").Children()
  99. if err == nil && arrow_dbCnts != nil {
  100. hostTypes = append(hostTypes, "arrow_db")
  101. }
  102. var component_ips map[string][]string = check.GetMap(configJson, hostTypes)
  103.  
  104. // Iterate over the go map collecting key's and value's to write the hosts to a file
  105. for prod_component, ips := range component_ips {
  106.  
  107. for x := 0; x < len(ips); x++ {
  108. ip := ips[x]
  109.  
  110. // Run the checkHost method in parallel
  111. go checkHost(sshConn, configJson, ch, prod_component, ip)
  112. numChannels++
  113. }
  114. }
  115.  
  116. // Store the result for each channel
  117. for i := 1; i <= numChannels; i++ {
  118. result := <-ch
  119. if result.name == "" {
  120. log.Errorf("Failed ", result.name)
  121. }
  122. }
  123.  
  124. log.Info("Complete!")
  125. }
  126.  
  127. // SSH and run the postCheck method on the current node
  128. func checkHost(sshConn sshutility.SSHConnection, configJson *gabs.Container, ch chan sshChan, prod_component string, hostIP string) {
  129.  
  130. node := map[string]string{
  131. "host": hostIP,
  132. }
  133.  
  134. // Point towards the Client in the ssh library
  135. sshClient, err := sshConn.Connect(node)
  136.  
  137. if err != nil {
  138. log.Fatalf("Error when connecting to node[%s(%s)]: %v", prod_component, hostIP, err)
  139. }
  140. defer sshClient.Close()
  141. postCheck(sshClient, configJson, prod_component, hostIP)
  142.  
  143. // Mark as done
  144. ch <- sshChan{
  145. name: hostIP,
  146. }
  147. }
  148.  
  149. // Method that runs role specific checks to ensure certain tasks/services are running
  150. func postCheck(sshClient *ssh.Client, configJson *gabs.Container, prod_component string, hostIP string) bool {
  151.  
  152. // Split up the component names into cluster and role
  153. prodCompArray := strings.Split(prod_component, ".")
  154. product, componentName := prodCompArray[0], prodCompArray[1]
  155.  
  156. // Get the arrowdb_url_prefix and domain_name form the user_input
  157. arrowdb_url_prefix, _ := configJson.Path("user_input.arrowdb_url_prefix").Data().(string)
  158. domain_name, _ := configJson.Path("user_input.domain_name").Data().(string)
  159.  
  160. // Switch cases to distinguish between the roles
  161. switch product {
  162. case "arrow_cloud":
  163. switch componentName {
  164. case "stratus":
  165.  
  166. curl := "curl --header \"Host: " + arrowdb_url_prefix + "." + domain_name + "\" " + hostIP + ":7000/ping.json"
  167. var expected = "\"ping\": \"ok\""
  168. runCurl(sshClient, curl, product, componentName, hostIP, expected)
  169.  
  170. case "log_aggregator":
  171.  
  172. case "app_haproxy":
  173.  
  174. curl := "curl --header \"Host: " + arrowdb_url_prefix + "." + domain_name + "\" " + hostIP + "/v1/admins/ping.json"
  175. var expected = "\"status\":\"ok\""
  176. runCurl(sshClient, curl, product, componentName, hostIP, expected)
  177.  
  178. case "admin_haproxy":
  179.  
  180. curl := "curl --header \"Host: " + arrowdb_url_prefix + "." + domain_name + "\" " + hostIP + "/v1/admins/ping.json"
  181. var expected = "\"status\":\"ok\""
  182. runCurl(sshClient, curl, product, componentName, hostIP, expected)
  183.  
  184. case "admin":
  185.  
  186. case "docker_registry":
  187.  
  188. case "container_node":
  189.  
  190. curl := "curl --header \"Host: " + arrowdb_url_prefix + "." + domain_name + "\" " + hostIP + ":9090/ping"
  191. containerSession, _ := sshClient.NewSession()
  192. out, _ := containerSession.Output(curl)
  193. result := string(out[:])
  194. defer containerSession.Close()
  195. var expected = "\"success\": true"
  196. compareOutput(result, curl, product, componentName, hostIP, expected)
  197.  
  198. default:
  199. log.Error("the hosts are unreadable")
  200. return false
  201. }
  202. case "arrow_db":
  203. switch componentName {
  204. case "api":
  205.  
  206. urlSession, _ := sshClient.NewSession()
  207. if err := urlSession.Run("curl " + hostIP + ":8083"); err != nil {
  208. fmt.Println("error")
  209. }
  210. defer urlSession.Close()
  211.  
  212. case "upload":
  213.  
  214. delayed_jobSession, _ := sshClient.NewSession()
  215. output, _ := delayed_jobSession.Output("ps -ef")
  216. outputString := string(output[:])
  217. defer delayed_jobSession.Close()
  218. if !strings.Contains(outputString, "delayed_job") {
  219. log.Error("delayed_job is not running on " + hostIP + " \n")
  220. }
  221.  
  222. urlSession, _ := sshClient.NewSession()
  223. if err := urlSession.Run("curl " + hostIP + ":8083"); err != nil {
  224. fmt.Println("error")
  225. }
  226. defer urlSession.Close()
  227.  
  228. case "admin":
  229.  
  230. case "haproxy":
  231.  
  232. curl := "curl --header \"Host: " + arrowdb_url_prefix + "." + domain_name + "\" " + hostIP + "/v1/admins/ping.json"
  233. var expected = "\"status\":\"ok\""
  234. runCurl(sshClient, curl, product, componentName, hostIP, expected)
  235.  
  236. case "push":
  237.  
  238. default:
  239. log.Println("the hosts are unreadable")
  240. return false
  241. }
  242. }
  243.  
  244. return true
  245. }
  246.  
  247. // Method to cmpare the output of the test with the expected result
  248. func compareOutput(result string, curl string, product string, componentName string, hostIP string, expected string) bool {
  249. // Check if the output contains the result
  250. if !strings.Contains(result, expected) {
  251. return false
  252. }
  253. return true
  254. }
  255.  
  256. // Method that ssh's into the host and runs a curl command
  257. func runCurl(sshClient *ssh.Client, curl string, product string, componentName string, hostIP string, expected string) {
  258. result := ""
  259. // Check if the ssh.Client is not nil
  260. if sshClient != nil {
  261. curlSession, _ := sshClient.NewSession()
  262. out, _ := curlSession.Output(curl)
  263. result = string(out[:])
  264. if result == "" {
  265. log.Error("cannot curl : " + curl)
  266. }
  267. defer curlSession.Close()
  268. }
  269.  
  270. // Compare the output with the expected result
  271. var output = compareOutput(result, curl, product, componentName, hostIP, expected)
  272. if output == false {
  273. log.Error("cannot "+curl+" from <"+product+"."+componentName+">, IP : ", hostIP)
  274. }
  275. }
  276.  
  277. /**
  278. If user specifies hosts array in user_input only (does not assign hosts to arrowcloud/DB roles) we should get
  279. configuration from database which was saved when setting up the cluster.
  280. */
  281. func getConfigJsonFromDatabase(configJson *gabs.Container) *gabs.Container {
  282.  
  283. dbUsername, _ := configJson.Path("user_input.mongodb.username").Data().(string)
  284. dbPassword, _ := configJson.Path("user_input.mongodb.password").Data().(string)
  285.  
  286. var mongoHosts []string
  287. helper.ExtractStringArray(&mongoHosts, configJson, "user_input.mongodb.hosts")
  288. var mongoHostPorts []string
  289. for x := 0; x < len(mongoHosts); x++ {
  290. mongoHostPorts = append(mongoHostPorts, mongoHosts[x]+":27017")
  291. }
  292.  
  293. log.Infof("Retrieving configuration from database: %v", mongoHostPorts)
  294.  
  295. // Initialize info using the username, password and Host
  296. mongoDBDialInfo := &mgo.DialInfo{
  297. Addrs: mongoHostPorts,
  298. Timeout: 10 * time.Second,
  299. Database: "deployment_stacks",
  300. Username: dbUsername,
  301. Password: dbPassword,
  302. }
  303.  
  304. // Dial into mongo db using the replica set
  305. session, err := mgo.DialWithInfo(mongoDBDialInfo)
  306. if err != nil {
  307. log.Fatalf("Failed to connect to mongo nodes: %s\n", err)
  308. }
  309. session.SetMode(mgo.Monotonic, true)
  310.  
  311. // Select the host_mapping collection
  312. c := session.DB("deployment_stacks").C("hosts_mapping")
  313.  
  314. result := HostsMappingDoc{}
  315.  
  316. // Find the record
  317. err = c.Find(bson.M{}).One(&result)
  318. if err != nil {
  319. log.Fatalf("Failed to get configuration from database. %v", err)
  320. }
  321.  
  322. // Create a json object for the host map
  323. configInMongo, err := gabs.Consume(result.Mapping)
  324. if err != nil {
  325. log.Fatalf("Failed to parsing configuration got from database. %v", err)
  326. }
  327.  
  328. return configInMongo
  329. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement