API tools faq
paste
Guest User

Untitled

a guest
Oct 15th, 2019
250
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
YAML 9.74 KB | None | 0 0
raw download clone embed print report
  1. Description: Deploying a Logstash service using the S3 output plugin, on AWS Fargate.
  2. Parameters:
  3.   EnvironmentName:
  4.     Type: String
  5.     Default: LogstashS3Environment
  6.     Description: The name of the environment to add this service to.
  7.   EnvironmentRegion:
  8.     Type: String
  9.     Default: eu-west-2
  10.   S3BucketName:
  11.     Type: String
  12.     Default: logstash-output-here3
  13.     Description: A name for the bucket to which logstash ships logs. Must be unique, to existing buckets. Lowercase only.
  14.   S3BucketCannedACL:
  15.     Type: String
  16.     Default: private
  17.     Description: A name for the bucket to which logstash ships logs. Must be unique, to existing buckets. Lowercase only.
  18.   LogstashClusterName:
  19.     Type: String
  20.     Default: LogstashCluster
  21.     Description: A name for the Cluster.
  22.   LogstashContainerSecurityGroupDescription:
  23.     Type: String
  24.     Default: SecurityGroup blocking all that are not on port 5044(UDP/Syslog) or 80(HealthChecks).
  25.   LogstashContainerName:
  26.     Type: String
  27.     Default: LogstashContainer
  28.     Description: A name for the container.
  29.   ServiceName:
  30.     Type: String
  31.     Default: LogstashService
  32.     Description: A name for the service.
  33.   ImageUrl:
  34.     Type: String
  35.     Default: kalledania/logstashimage:latest
  36.     Description: The url of our custom Logstash docker image.
  37.   SyslogInputPort:
  38.     Type: Number
  39.     Default: 5044
  40.     Description: What port number the application inside the docker container is binding to.
  41.   S3BucketRegion:
  42.     Type: String
  43.     Default: eu-west-2
  44.     Description: The region of the S3 bucket we want to ship logs to.
  45.   NACLAllowedIPS:
  46.     Type: String
  47.     Default: 0.0.0.0/0
  48.     Description: You should probably change this to the specific IP you expect to recieve logs from.
  49.   ContainerCPU:
  50.     Type: String
  51.     Default: 256
  52.     Description: How much CPU to give the container. 1024 is 1 CPU.
  53.   ContainerMemory:
  54.     Type: Number
  55.     Default: 1024
  56.     Description: How much memory in megabytes to give the container. I recommend not going lower than 1024, as 512 occationally experienced out-of-memory errors on container start-up.
  57.   DesiredCount:
  58.     Type: Number
  59.     Default: 1
  60.     Description: How many copies of the service task to run.
  61.   VPCCIDR:
  62.     Type: String
  63.     Default: 10.0.0.0/16
  64.   Subnet1Range:
  65.     Type: String
  66.     Default: 10.0.0.0/24
  67.   Subnet2Range:
  68.     Type: String
  69.     Default: 10.0.1.0/24
  70.   ContainerAllowInboundIPs:
  71.     Type: String
  72.     Default: 0.0.0.0/0
  73.     Description: You should probably change this to the specific IP you expect to recieve logs from.
  74.   AssignPublicIP:
  75.     Type: String
  76.     Default: ENABLED
  77.  
  78. Resources:
  79.   LogstashBucket:
  80.     Type: AWS::S3::Bucket
  81.     Properties:
  82.       BucketName: !Ref S3BucketName
  83.       BucketEncryption:
  84.         ServerSideEncryptionConfiguration:
  85.           -
  86.             ServerSideEncryptionByDefault:
  87.               SSEAlgorithm: AES256
  88.       PublicAccessBlockConfiguration:
  89.         BlockPublicAcls: false #true
  90.         BlockPublicPolicy: false #true
  91.         IgnorePublicAcls: false #true
  92.         RestrictPublicBuckets: false #true
  93.       AccessControl: PublicReadWrite #Private
  94.   # Endpoints provide direct access to a service inside the same AWS account, which means our traffic will never reach the public world wide web. We can even use Endpoints to provide access from a private.
  95.   EndpointS3:
  96.     Type: AWS::EC2::VPCEndpoint
  97.     Properties:
  98.       VpcId: !Ref LogstashVPC
  99.       ServiceName: !Sub com.amazonaws.${AWS::Region}.s3
  100.       RouteTableIds:
  101.       - !Ref LogstashRouteTable
  102.       VpcEndpointType: "Gateway"
  103.   #LogstashBucketPolicyPublicReadFromIP:
  104.  #   Type: AWS::S3::BucketPolicy
  105.   #  Properties:
  106.   #    Bucket: !Ref LogstashBucket
  107.   #    PolicyDocument:
  108.   #      Statement:
  109.    #     -
  110.   #        Effect: "Allow"
  111.            #Allow only the referenced Arn to PutObject to this bucket..
  112.   #        Action: "s3:PutObject"
  113.    #       Resource:
  114.   #          Fn::Join:
  115.   #          - ""
  116.   #          -   - "arn:aws:s3:::"
  117.   #              - !Ref LogstashBucket
  118.   #              - "/*"  
  119.   #        Principal: '*' #!Ref LogstashRolePutToS3
  120.  #         Condition:
  121.   #          ArnEquals:
  122.   #            aws:SourceArn: !Ref LogstashTaskDef #LogstashRolePutToS3  
  123.   LogstashPolicyPutToS3:
  124.     Type: AWS::IAM::ManagedPolicy
  125.     Properties:
  126.       ManagedPolicyName: "LogstashPolicyPutToS3"
  127.       PolicyDocument:
  128.         Version: "2012-10-17"
  129.         Statement:
  130.         -
  131.           Effect: "Allow"
  132.           Action: "s3:PutObject"
  133.           Resource: !GetAtt LogstashBucket.Arn    
  134.   LogstashRolePutToS3:
  135.     Type: AWS::IAM::Role
  136.     Properties:
  137.       AssumeRolePolicyDocument:
  138.         Version: "2012-10-17"
  139.         Statement:
  140.          -
  141.             Effect: "Allow"
  142.             Principal:
  143.               Service:
  144.                 - "ecs-tasks.amazonaws.com"
  145.             Action: "sts:AssumeRole"
  146.       ManagedPolicyArns:
  147.         - !Ref LogstashPolicyPutToS3
  148.       RoleName: "LogstashRolePutToS3"
  149.   LogstashVPC:
  150.     Type: AWS::EC2::VPC
  151.     Properties:
  152.       CidrBlock: !Ref VPCCIDR
  153.       EnableDnsHostnames: "true"
  154.       EnableDnsSupport: "true"
  155.       InstanceTenancy: "default"
  156.   LogstashSub1:
  157.     Type: AWS::EC2::Subnet
  158.     Properties:
  159.       CidrBlock: !Ref Subnet1Range
  160.       MapPublicIpOnLaunch: "true"
  161.       VpcId: !Ref LogstashVPC
  162.   LogstashSub2:
  163.     Type: AWS::EC2::Subnet
  164.     Properties:
  165.       CidrBlock: !Ref Subnet2Range
  166.       MapPublicIpOnLaunch: "true"
  167.       VpcId: !Ref LogstashVPC
  168.   LogstashInternetGateway:
  169.     Type: AWS::EC2::InternetGateway
  170.   LogstashInternetGatewayAttachAction:
  171.     Type: AWS::EC2::VPCGatewayAttachment
  172.     Properties:
  173.       InternetGatewayId: !Ref LogstashInternetGateway  
  174.       VpcId: !Ref LogstashVPC
  175.   LogstashRouteTableAssociation1:
  176.     Type: AWS::EC2::SubnetRouteTableAssociation
  177.     Properties:
  178.       SubnetId: !Ref LogstashSub1
  179.       RouteTableId: !Ref LogstashRouteTable
  180.   LogstashRouteTableAssociation2:
  181.     Type: AWS::EC2::SubnetRouteTableAssociation
  182.     Properties:
  183.       SubnetId: !Ref LogstashSub2
  184.       RouteTableId: !Ref LogstashRouteTable
  185.   LogstashRouteTable:                      
  186.     Type: AWS::EC2::RouteTable
  187.     Properties:
  188.       VpcId: !Ref LogstashVPC
  189.   LogstashVPCInternetGatewayRoute:
  190.     Type: AWS::EC2::Route
  191.     Properties:
  192.       DestinationCidrBlock: 0.0.0.0/0
  193.       GatewayId: !Ref LogstashInternetGateway
  194.       RouteTableId: !Ref LogstashRouteTable
  195.   LogstashCluster:
  196.     Type: AWS::ECS::Cluster
  197.     Properties:
  198.       ClusterName: !Ref LogstashClusterName
  199.   LogGroup:
  200.     Type: AWS::Logs::LogGroup
  201.     Properties:
  202.       LogGroupName: !Join ['', [/ecs/, !Ref ServiceName, TaskDefinition]]
  203.   PolicyECSExecution:
  204.     Type: AWS::IAM::ManagedPolicy
  205.     Properties:
  206.       ManagedPolicyName: "PolicyECSExecution"
  207.       PolicyDocument:
  208.         Version: "2012-10-17"
  209.         Statement:
  210.         -
  211.           Effect: "Allow"
  212.           Action:
  213.           - ecr:GetAuthorizationToken
  214.           - ecr:BatchCheckLayerAvailability
  215.           - ecr:GetDownloadUrlForLayer
  216.           - ecr:BatchGetImage
  217.           - logs:CreateLogStream
  218.           - logs:PutLogEvents
  219.           Resource: '*'
  220.   RoleECSExecution:
  221.     Type: AWS::IAM::Role
  222.     Properties:
  223.       AssumeRolePolicyDocument:
  224.         Version: "2012-10-17"
  225.         Statement:
  226.          -
  227.             Effect: "Allow"
  228.             Principal:
  229.               Service:
  230.                 - "ecs-tasks.amazonaws.com"
  231.             Action: "sts:AssumeRole"
  232.       ManagedPolicyArns:
  233.         - !Ref PolicyECSExecution
  234.       RoleName: "RoleECSExecution"
  235.   LogstashTaskDef:
  236.     Type: AWS::ECS::TaskDefinition
  237.     Properties:
  238.       NetworkMode: awsvpc
  239.       RequiresCompatibilities:
  240.         - FARGATE
  241.       Cpu: !Ref ContainerCPU
  242.       Memory: !Ref ContainerMemory  
  243.       ExecutionRoleArn: !Ref RoleECSExecution                                      
  244.       TaskRoleArn: !Ref LogstashRolePutToS3
  245.       ContainerDefinitions:
  246.         -
  247.           Name: !Ref LogstashContainerName
  248.           Image: !Ref ImageUrl
  249.           PortMappings:
  250.            -
  251.               ContainerPort: !Ref SyslogInputPort
  252.               HostPort: !Ref SyslogInputPort
  253.           LogConfiguration:
  254.             LogDriver: awslogs
  255.             Options:
  256.               awslogs-region: !Ref AWS::Region
  257.               awslogs-group: !Ref LogGroup
  258.               awslogs-stream-prefix: ecs
  259.           Environment:
  260.             - Name: S3_BUCKET_BUCKETNAME
  261.               Value: !Ref S3BucketName
  262.             - Name: S3_BUCKET_CANNED_ACL
  263.               Value: !Ref S3BucketCannedACL
  264.             - Name: SYSLOG_INPUT_PORT
  265.               Value: !Ref SyslogInputPort
  266.             - Name: S3_BUCKET_REGION
  267.               Value: !Ref S3BucketRegion  
  268.   LogstashContainerSecurityGroup:                                    
  269.     Type: AWS::EC2::SecurityGroup
  270.     Properties:
  271.       GroupDescription: !Ref LogstashContainerSecurityGroupDescription
  272.       VpcId: !Ref LogstashVPC
  273.       SecurityGroupIngress:
  274.         - IpProtocol: tcp
  275.           CidrIp: !Ref ContainerAllowInboundIPs
  276.           FromPort: !Ref SyslogInputPort
  277.           ToPort: !Ref SyslogInputPort
  278.         - IpProtocol: tcp
  279.           CidrIp: !Ref ContainerAllowInboundIPs
  280.           FromPort: 80
  281.           ToPort: 80
  282.   LogstashService:
  283.     Type: AWS::ECS::Service
  284.     Properties:
  285.       ServiceName: !Ref ServiceName
  286.       Cluster: !Ref LogstashCluster
  287.       TaskDefinition: !Ref LogstashTaskDef
  288.       DesiredCount: !Ref DesiredCount
  289.       LaunchType: FARGATE
  290.       NetworkConfiguration:
  291.         AwsvpcConfiguration:
  292.           AssignPublicIp: !Ref AssignPublicIP
  293.           Subnets:
  294.            - !Ref LogstashSub1
  295.             - !Ref LogstashSub2
  296.           SecurityGroups:
  297.            - !Ref LogstashContainerSecurityGroup
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!