Guest User

Untitled

a guest
Oct 23rd, 2017
135
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.97 KB | None | 0 0
  1. #
  2. # robots.txt
  3. #
  4. # This file is to prevent the crawling and indexing of certain parts
  5. # of your site by web crawlers and spiders run by sites like Yahoo!
  6. # and Google. By telling these "robots" where not to go on your site,
  7. # you save bandwidth and server resources.
  8. #
  9. # This file will be ignored unless it is at the root of your host:
  10. # Used: http://example.com/robots.txt
  11. # Ignored: http://example.com/site/robots.txt
  12. #
  13. # For more information about the robots.txt standard, see:
  14. # http://www.robotstxt.org/robotstxt.html
  15.  
  16. User-agent: *
  17. # CSS, JS, Images
  18. Allow: /core/*.css$
  19. Allow: /core/*.css?
  20. Allow: /core/*.js$
  21. Allow: /core/*.js?
  22. Allow: /core/*.gif
  23. Allow: /core/*.jpg
  24. Allow: /core/*.jpeg
  25. Allow: /core/*.png
  26. Allow: /core/*.svg
  27. Allow: /profiles/*.css$
  28. Allow: /profiles/*.css?
  29. Allow: /profiles/*.js$
  30. Allow: /profiles/*.js?
  31. Allow: /profiles/*.gif
  32. Allow: /profiles/*.jpg
  33. Allow: /profiles/*.jpeg
  34. Allow: /profiles/*.png
  35. Allow: /profiles/*.svg
  36. # Directories
  37. Disallow: /core/
  38. Disallow: /profiles/
  39. # Files
  40. Disallow: /README.txt
  41. Disallow: /web.config
  42. # Paths (clean URLs)
  43. Disallow: /admin
  44. Disallow: /comment/reply
  45. Disallow: /filter/tips
  46. Disallow: /node/add
  47. Disallow: /search
  48. Disallow: /recherche
  49. Disallow: /user
  50. Disallow: /utilisateur
  51. Disallow: /group/
  52. Disallow: /store/
  53. Disallow: /profile
  54. Disallow: /question
  55. Disallow: /page
  56. Disallow: /guide
  57. Disallow: /demande
  58. Disallow: /flag
  59. Disallow: /taxonomy
  60. Disallow: /cart
  61. # Paths (no clean URLs)
  62. Disallow: /index.php/admin
  63. Disallow: /index.php/comment/reply
  64. Disallow: /index.php/filter/tips
  65. Disallow: /index.php/node/add
  66. Disallow: /index.php/search
  67. Disallow: /index.php/recherche
  68. Disallow: /index.php/user
  69. Disallow: /index.php/utilisateur
  70. Disallow: /index.php/group/
  71. Disallow: /index.php/store/
  72. Disallow: /index.php/profile
  73. Disallow: /index.php/question
  74. Disallow: /index.php/page
  75. Disallow: /index.php/guide
  76. Disallow: /index.php/demande
  77. Disallow: /index.php/flag
  78. Disallow: /index.php/taxonomy
  79. Disallow: /index.php/cart
  80. # Sitemaps
  81. Sitemap: https://www.s1biose.com/sitemap.xml
Add Comment
Please, Sign In to add comment