Add an EFS volume into Secure Agent infrastructure

This volume will act as the persistence storage for Secure Agent's logs and configurations.

.gitignore

@@ -12,3 +12,4 @@ sandbox
 *.tfvars
 .idea/
 *.backup
+*.tfplan

README.md

@@ -122,8 +122,8 @@ See Terraform doc on [variables](https://www.terraform.io/docs/configuration/var
 image can be used for production set up.
 
 ### Credentials in Terraform
-* It's recommended that to avoid having Informatica credentials in bash history, all the variables can be defined in a `*.tfvars` 
-file and pass to terraform using `-var-file` argument to terraform.  
+* It's recommended to define all variables values in a `*.tfvars` file and pass that to terraform using `-var-file` argument.
+ This will avoid having Informatica credentials in bash history. 
 
 ```shell script
 $ cd terraform

terraform/autoscaling-group.tf

+resource "aws_autoscaling_group" "secure-agent-autoscaling-group" {
+  # as per our current licesning in IICS, each docker instance that
+  # runs on EC2 will treat at as a new license.
+  name = "secure-agent-autoscaling-group"
+  desired_capacity = 1
+  max_size = 1
+  min_size = 1
+
+  vpc_zone_identifier = data.aws_subnet_ids.subnets.ids
+  health_check_type = "ELB"
+  launch_configuration = aws_launch_configuration.secure-agent-launch-config.name
+}
+
+data "aws_ami" "ecs-optimized" {
+  most_recent = true
+  owners      = ["amazon"]
+
+  filter {
+    name   = "name"
+    values = ["*-amazon-ecs-optimized"]
+  }
+}
+
+resource "aws_launch_configuration" "secure-agent-launch-config" {
+  name = "secure-agnet-launch-configuration"
+  image_id      = data.aws_ami.ecs-optimized.image_id
+  placement_tenancy = "default"
+
+  iam_instance_profile = aws_iam_instance_profile.ecs-instance-profile.name
+  security_groups = [data.aws_security_group.sec-group.id]
+  user_data = <<EOF
+              #!/bin/bash
+              echo ECS_CLUSTER=${var.ecs_cluster_name} >> /etc/ecs/ecs.config
+              EOF
+  instance_type = var.instance_type
+}
\ No newline at end of file

terraform/ecs.tf

 data "template_file" "container" {
-  template = file("./templates/container.tpl")
+  template = file("./templates/container-definitions.tpl")
   vars     = {
-    container_name       = var.container_name
-    image_name           = var.image_name
-    fargate_cpu          = var.fargate_cpu
-    fargate_memory       = var.fargate_memory
-    app_port1            = var.container_app_port[0]
-    app_port2            = var.container_app_port[1]
-    app_port3            = var.container_app_port[2]
-    network_mode         = var.container_network_mode
-    informatica_username = var.informatica_username
-    informatica_password = var.informatica_password
+    container_name          = var.container_name
+    image_name              = var.image_name
+    app_port1               = var.container_app_port[0]
+    app_port2               = var.container_app_port[1]
+    app_port3               = var.container_app_port[2]
+    informatica_username    = var.informatica_username
+    informatica_password    = var.informatica_password
+    volume1                 = var.secure_agnet_container_volumes[0]
+    volume2                 = var.secure_agnet_container_volumes[1]
+    volume3                 = var.secure_agnet_container_volumes[2]
+    volume4                 = var.secure_agnet_container_volumes[3]
+    secure_agent_efs_volume = var.secure_agent_efs_volume
   }
 }
 
@@ -23,10 +25,23 @@ resource "aws_ecs_task_definition" "task" {
   execution_role_arn       = data.aws_iam_role.ecs-task-execution.arn
   network_mode             = var.container_network_mode
   requires_compatibilities = [
-    "FARGATE"]
-  cpu                      = var.fargate_cpu
-  memory                   = var.fargate_memory
+    "EC2"]
   container_definitions    = data.template_file.container.rendered
+  volume {
+    name      = var.secure_agent_efs_volume
+    host_path = ""
+    docker_volume_configuration {
+      autoprovision = true
+      scope         = "shared"
+
+      driver_opts = {
+        "type"   = "nfs"
+        "device" = "${aws_efs_file_system.secure-agent-fs.dns_name}:/"
+        "o"      = "addr=${aws_efs_file_system.secure-agent-fs.dns_name},nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,nosuid"
+      }
+    }
+  }
+  tags                     = var.ecs_task_tags
 }
 
 resource "aws_ecs_service" "service" {
@@ -34,7 +49,8 @@ resource "aws_ecs_service" "service" {
   cluster         = aws_ecs_cluster.cluster.id
   task_definition = aws_ecs_task_definition.task.arn
   desired_count   = 1
-  launch_type     = "FARGATE"
+  # secure agent configs and logs are persisted into an EFS volume.
+  launch_type     = "EC2"
 
   network_configuration {
     security_groups  = [

terraform/efs.tf

+resource "aws_efs_file_system" "secure-agent-fs" {
+  tags = var.efs_tags
+}
+
+output "aws_efs_token" {
+  value = aws_efs_file_system.secure-agent-fs.creation_token
+}
+
+resource "aws_efs_mount_target" "secure-agent-fs-mount" {
+  file_system_id  = aws_efs_file_system.secure-agent-fs.id
+  subnet_id       = sort(data.aws_subnet_ids.subnets.ids)[0]
+  security_groups = [
+    data.aws_security_group.sec-group.id]
+}
\ No newline at end of file

terraform/iam.tf

+# define a policy document for role below
+data "aws_iam_policy_document" "ecs-agent" {
+  statement {
+    actions = [
+      "sts:AssumeRole"]
+    principals {
+      type        = "Service"
+      identifiers = [
+        "ec2.amazonaws.com"]
+    }
+  }
+}
+
+# define the role for ECS agent so that ECS container agent can make API calls
+resource "aws_iam_role" "ecs-agent" {
+  name               = var.ecs_iam_role
+  assume_role_policy = data.aws_iam_policy_document.ecs-agent.json
+}
+
+# grant role permission for ECS agent operations
+resource "aws_iam_role_policy_attachment" "ecs-agent" {
+  role       = aws_iam_role.ecs-agent.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role"
+}
+
+# allow instance profile to assume this role
+resource "aws_iam_instance_profile" "ecs-instance-profile" {
+  name = "secure-agent-ecs-instance-profile"
+  role = aws_iam_role.ecs-agent.name
+}
\ No newline at end of file

terraform/network.tf

@@ -9,9 +9,4 @@ data "aws_subnet_ids" "subnets" {
     values = [
       var.private_subnets_filter["Name"]]
   }
-}
-
-data "aws_security_group" "sec-group" {
-  vpc_id = data.aws_vpc.vpc.id
-  tags   = var.security_group
 }
\ No newline at end of file

terraform/security.tf

-data "aws_security_group" "secgroup" {
+data "aws_security_group" "sec-group" {
   vpc_id = data.aws_vpc.vpc.id
   tags   = var.security_group
+}
+
+// open port 2049 for NFSv4
+resource "aws_security_group" "secure-agent-fs-" {
+  name   = "secure-agent-efs-sg"
+  vpc_id = data.aws_vpc.vpc.id
+
+  // NFS
+  ingress {
+    security_groups = [
+      data.aws_security_group.sec-group.id]
+    from_port       = 2049
+    to_port         = 2049
+    protocol        = "tcp"
+  }
+
+  egress {
+    security_groups = [
+      data.aws_security_group.sec-group.id]
+    from_port       = 0
+    to_port         = 0
+    protocol        = "-1"
+  }
+
+  tags = var.secure_agent_sg_tags
 }
\ No newline at end of file

terraform/templates/container.tpl → terraform/templates/container-definitions.tpl

@@ -2,9 +2,7 @@
   {
     "name": "${container_name}",
     "image": "${image_name}",
-    "cpu": ${fargate_cpu},
-    "memory": ${fargate_memory},
-    "networkMode": "${network_mode}",
+    "memory": 4096,
     "portMappings": [
       {
         "containerPort": ${app_port1},
@@ -28,6 +26,24 @@
             "name": "INFORMATICA_PASSWORD",
             "value": "${informatica_password}"
         }
+    ],
+    "mountPoints": [
+        {
+            "containerPath": "${volume1}",
+            "sourceVolume": "${secure_agent_efs_volume}"
+        },
+        {
+            "containerPath": "${volume2}",
+            "sourceVolume": "${secure_agent_efs_volume}"
+        },
+        {
+            "containerPath": "${volume3}",
+            "sourceVolume": "${secure_agent_efs_volume}"
+        },
+        {
+            "containerPath": "${volume4}",
+            "sourceVolume": "${secure_agent_efs_volume}"
+        }
     ]
   }
 ]
\ No newline at end of file

terraform/variables.tf

+variable "informatica_username" {}
+variable "informatica_password" {}
+
 variable "aws_shared_cred_file" {}
 variable "aws_profile" {
   default = "default"
@@ -57,18 +60,17 @@ variable "container_count" {
   default = 1
 }
 
-# https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html
-# see Secure Agent resource requirements for these numbers.
-variable "fargate_cpu" {
-  # 1 vCPU = 1024 CPU units
-  default = "4096"
-}
-variable "fargate_memory" {
-  # in MiB
-  default = "8192"
+# see Secure Agent system requirements
+variable "instance_type" {
+  default = "a1.xlarge"
 }
 
 # ecs
+
+variable "ecs_iam_role" {
+  default = "iics-secure-agent-iam-role"
+}
+
 variable "ecs_cluster_name" {
   default = "iics-agent-cluster"
 }
@@ -85,9 +87,44 @@ variable "ecs_task_name" {
   default = "iics-agent-task"
 }
 
+variable "ecs_task_tags" {
+  type    = map(string)
+  default = {
+    Name = "iics-secure-agent"
+    tier = "test"
+  }
+}
+
 variable "ecs_service_name" {
   default = "iics-agent-service"
 }
 
-variable "informatica_username" {}
-variable "informatica_password" {}
\ No newline at end of file
+variable "efs_tags" {
+  type = map(string)
+  default = {
+    Name = "iics-secure-agent"
+    tier = "test"
+  }
+}
+
+variable "secure_agnet_container_volumes" {
+  type = list(string)
+  default = [
+    # see Dockerfile for these default values.
+    "/home/agent/infaagent/apps/agentcore/infaagent.log",
+    "/home/agent/infaagent/apps/agentcore/agentcore.log",
+    "/home/agent/infaagent/apps/agentcore/logs",
+    "/home/agent/infaagent/apps/agentcore/data"
+  ]
+}
+
+variable "secure_agent_efs_volume" {
+  default = "agent"
+}
+
+variable "secure_agent_sg_tags" {
+  default = {
+    Name = "secure-agent-efs-sg"
+    tier = "test"
+  }
+}
\ No newline at end of file