Merge branch 'odbc' into 'master'

ODBC, Log Directories, Docker-compose.yml update -INPLATFORM-283

See merge request !24

Dockerfile

@@ -18,7 +18,7 @@ ENV REGION=$REGION
 
 ARG AGENT_URL="https://${POD}.${REGION}.informaticacloud.com/saas/download/installer/linux64/agent64_install_ng_ext.bin"
 ARG USER=agent
-ARG WORK_DIR=/home/$USER/infaagent/apps/agentcore
+ENV INFA_HOME=/home/$USER/infaagent
 
 # install system tools
 RUN apt-get update && apt-get install -y \
@@ -28,7 +28,9 @@ locales \
 locales-all \
 sudo \
 unzip \
-jq
+jq \
+sqlite3 \
+libsqliteodbc
 
 # Set the locale, Locale defaults are necessary for agent to operate correctly
 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
@@ -42,6 +44,24 @@ RUN useradd --create-home -ms /bin/bash -U $USER
 RUN mkdir /exports && chown $USER:$USER /exports
 USER $USER
 
+# Add the SQLite ODBC information.  The main odbcinst.ini is in INFA_HOME directory.
+ADD odbcinst.ini /home/agent/.odbcinst.ini
+
+# Create directories so that volume mounts pick up the correct users
+RUN mkdir -p ~/databases && \
+  mkdir -p ~/data && \
+  mkdir -p $INFA_HOME/apps/Administrator/logs && \
+  mkdir -p $INFA_HOME/apps/agentcore/logs && \
+  mkdir -p $INFA_HOME/apps/CIHProcessor/logs && \
+  mkdir -p $INFA_HOME/apps/Common_Integration_Components/logs && \
+  mkdir -p $INFA_HOME/apps/Data_Integration_Server/logs && \
+  mkdir -p $INFA_HOME/apps/FileIntegrationService/logs && \
+  mkdir -p $INFA_HOME/apps/MassIngestionRuntime/logs && \
+  mkdir -p $INFA_HOME/apps/OpsInsightsDataCollector/logs && \
+  mkdir -p $INFA_HOME/apps/process-engine/logs && \
+  mkdir -p $INFA_HOME/apps/Data_Integration_Server/ext/drivers && \
+  mkdir -p $INFA_HOME/apps/agentcore/conf
+
 # 1. Download and prepare Installer
 # 2. Set file permissions
 # 3. Install using silent install and the default location
@@ -51,7 +71,7 @@ chmod +x /tmp/agent64_install.bin && \
 /tmp/agent64_install.bin -i silent && \
 rm -rf /tmp/agent64_install.bin
 
-WORKDIR $WORK_DIR
+WORKDIR $INFA_HOME/apps/agentcore
 
 COPY run_agent.sh .
 

README.md

@@ -13,7 +13,7 @@
     * [Known Issues](#known-issues)
     * [Enterprise Integration Secure Agent](#enterprise-integration-secure-agent)
         * [Architecture Diagram](#architecture-diagram)
-        * [CI/CD and Terraform](#cicd-and-terraform) 
+        * [CI/CD and Terraform](#cicd-and-terraform)
         * [Informatica User](#informatica-user)
         * [Credentials in Terraform](#credentials-in-terraform)
             * [Test Environment](#test-environment)
@@ -22,14 +22,14 @@
         * [EFS Mounting, Accessing Secure Agent Configurations, Logs and Additional Debugging](#efs-mounting-accessing-secure-agent-configurations-logs-and-additional-debugging)
 
 # User Guide
-[userguide]: #user-guide 
+[userguide]: #user-guide
 User guide for AWS hosted Secure Agent is available [here](./userguide.md).
 
 # IICS Secure Agent Docker Image
 [dockerimage]: #iics-secure-agent-docker-image
 This document covers following topics:
 1. How ro run IICS Secure Agent using Docker image.
-2. IICS Secure Agent deployed(including Terraform configuration) in AWS(interop). 
+2. IICS Secure Agent deployed(including Terraform configuration) in AWS(interop).
 
 ## Description
 [description]: #description
@@ -59,35 +59,50 @@ The container executes a bash script called run_agent.sh upon start.  It needs I
 * INFORMATICA_PASSWORD (required) - Password for above credential.  Used for API access only.
 * JSON_LOG (optional) - If this is set, then the log output will be in JSON format.
 
-### Externalized Configurations
+**It is best practice to put these environment variables in a Docker [.env](https://docs.docker.com/compose/env-file/) file.**
+
+
+### Persisting Secure Agent Configuration With Docker Volumes
 [configs]: #externalized-configurations
-Secure Agent's configurations can be externalized using Docker [volumes](https://docs.docker.com/storage/volumes/). Following 
-Secure Agent's directories and files can be externalized for containers. Note that after initial 
-container startup, these configurations can be used to start subsequent containers.  
-
-* `agentcore.log` - Secure Agent default runtime log file. 
-* `infaagent.log` - Secure Agent start up log file.
-* `logs` - contains Secure Agent runtime log files and lock files.
-* `data` - contains the configuration data for mapping and integration components.
-* `conf` - contains Secure Agent runtime configuration files.
-   1. `infaagent.ini` - contains the registration information for the secure agent(this will be populated by Secure Agent during start up).
-   2. `proxy.ini` - proxy configurations for Secure Agent, if running behind a [proxy](https://docs.informatica.com/integration-cloud/cloud-platform/current-version/administrator/runtime-environments/secure-agents/configuring-a-proxy-to-exclude-non-proxy-hosts.html) server.  
-
-Following ports(among others) in Secure Agents can be mapped to host for externalized control. 
-* `7080` - Process Engine shutdown port.
-* `7443` - Process Engine https port.
-* `5432` - Process Engine Postgres DB port.
+Once the container starts and registers with Informatica API using the INFORMATICA_USER and INFORMATICA_PASSWORD credentials, a file will be created in the INFA_HOME/apps/agentcore/conf directory called `infaagent.ini`.  This file contains the organization and agent ids and the registration token and should be persisted, so that when a container is restarted or redeployed, the configuration will be saved.
+
+In addition to infagent.ini, there are additional files in the INFA_HOME/apps/agentcore/conf directory that you may want to persist, including `proxy.ini` used when running behind a [proxy](https://docs.informatica.com/integration-cloud/cloud-platform/current-version/administrator/runtime-environments/secure-agents/configuring-a-proxy-to-exclude-non-proxy-hosts.html) server.
+
+To persist these configuration files, use Docker [volumes](https://docs.docker.com/storage/volumes/) to either mount individual files or the conf directory into the Docker container.
+
+### Logs and Log Directories
+
+The Secure Agent parent process has a startup log in INFA_AGENT/apps/agentcore.log and port and Tomcat logs in INFA_AGENT/apps/agentcore/logs.
+
+The following directories contain log files for each of the IICS applications:
+* INFA_HOME/apps/Administrator/logs
+* INFA_HOME/apps/CIHProcessor/logs
+* INFA_HOME/apps/Common_Integration_Components/logs
+* INFA_HOME/apps/Data_Integration_Server/logs
+* INFA_HOME/apps/FileIntegrationService/logs
+* INFA_HOME/apps/MassIngestionRuntime/logs
+* INFA_HOME/apps/OpsInsightsDataCollector/logs
+* INFA_HOME/apps/process-engine/logs
+
+The above file and directories can also be persisted using Docker volumes.
+
+### Ports
+There are several ports in use by the secure agent, although it isn't strictly necessary to allow access to these ports in most cases.
+
+  * `7005` - Tomcat Shutdown Port
+  * `7080` and `7443` - Process Engine HTTP and HTTPS ports used to call the [Secure Agent as a Platform](https://kb.informatica.com/howto/6/pages/19/507918.aspx) (AAP) directly.  In other words, Application Integration Processes that you have created can be called on a specific Secure Agent by URL, e.g. https://your_agent_IP:7443/process-engine/public/rt/LookupInvoiceAndDetails?invoiceID=123.
+* `5432` - Process Engine PostgreSQL DB port.  Used for [load balanced Secure Agent groups](https://network.informatica.com/servlet/JiveServlet/download/17437-2-47463/Cloud+Application+Integration+-+Process+Server+Load+Balancing+and+Clustering+on+Secure+Agent.pdf) that must all connect to the same master database.
 
 See below section for examples on how to use volume and port mapping.
 
 ### Starting
 [starting]: #starting
-* Setting the hostname will provide the associated name in the IICS website. 
-* Note: since anybody who has access to see the processes can view the values `INFORMATICA_USER` and `INFORMATICA_PASSWORD`, it's 
-recommenced to configure them in a Docker [.env](https://docs.docker.com/compose/env-file/) file.
+* Setting the hostname will provide the associated name in the IICS website.
+* Note: since anybody who has access to see the processes can view the values `INFORMATICA_USER` and `INFORMATICA_PASSWORD`, it's recommenced to configure them in a Docker [.env](https://docs.docker.com/compose/env-file/) file.
+
 
 ```shell
-$ touch agentcore.log infaagent.log 
+$ touch agentcore.log infaagent.log
 $ docker run -d \
     -h <hostname> \
     --env INFORMATICA_USER=xxxxx \
@@ -95,16 +110,15 @@ $ docker run -d \
     -v $(pwd)/infaagent.log:/home/agent/infaagent/apps/agentcore/infaagent.log  \
     -v $(pwd)/agentcore.log:/home/agent/infaagent/apps/agentcore/agentcore.log  \
     -v $(pwd)/logs:/home/agent/infaagent/apps/agentcore/logs  \
-    -v $(pwd)/data:/home/agent/infaagent/apps/agentcore/data \
     -p 7080:7080 \
     -p 7443:7443 \
-    -p 5432:5432 \ 
+    -p 5432:5432 \
     --name <container_name> iics_secure_agent:<tag>
 ```
 
 * Note that if `conf` directory needs mounting, a minimum configuration for starting the Secure Agent is required, see [conf](./conf) for example.
 ```shell
-$ touch agentcore.log infaagent.log 
+$ touch agentcore.log infaagent.log
 $ docker run -d \
     -h <hostname> \
     --env INFORMATICA_USER=xxxxx \
@@ -116,16 +130,36 @@ $ docker run -d \
     -v $(pwd)/conf:/home/agent/infaagent/apps/agentcore/conf \
     -p 7080:7080 \
     -p 7443:7443 \
-    -p 5432:5432 \ 
+    -p 5432:5432 \
     --name <container_name> iics_secure_agent:<tag>
-```  
+```
+
+#### Using Docker Compose
+
+Create a .env file with the following entries.
+
+```
+INFORMATICA_USER=
+INFORMATICA_PASSWORD=
+HOSTNAME=
+CONFIG_DIR=
+LOG_DIR=
+ODBC_DATABASE_DIR=
+APPS_DIR=/home/agent/infaagent/apps
+```
+
+The included docker-compose.yml file uses the above environment variables to map directories into the container to persist log directories and infaagent.ini and odbc.ini files.  Modify for your use.
+
+```shell
+docker-compose up
+```
 
 ### Monitoring
 [monitor]: #monitoring
-If volume mapping is not used use `docke exec` to attach to the running container.
+If volume mapping is not used use `docker exec` to attach to the running container.
 
 ```shell
-docker exec -it <container_name> less agentCore.log
+docker exec -it <container_name> less agentcore.log
 ```
 
 ## Hardware Requirements
@@ -161,32 +195,32 @@ The diagram source can be found [here](https://www.lucidchart.com/documents/edit
 
 ### CI/CD and Terraform
 [terraform]: #cicd-and-terraform
-* Infrastructure is available as code in terraform for `AWS` provider, and [.gitlab-ci.yml](.gitlab-ci.yml) pipeline push the Secure Agent 
-image into [AWS ECR](https://aws.amazon.com/ecr/). 
-* Before disposing and create a new instance of Informatica Secure Agent, existing live connections to various targets(for e.g. databases) 
+* Infrastructure is available as code in terraform for `AWS` provider, and [.gitlab-ci.yml](.gitlab-ci.yml) pipeline push the Secure Agent
+image into [AWS ECR](https://aws.amazon.com/ecr/).
+* Before disposing and create a new instance of Informatica Secure Agent, existing live connections to various targets(for e.g. databases)
 should be reviewed and terminated. Due to this reason this is not part of CI/CD pipeline and can be executed as per necessity. Also, back up any logs and configurations
 before next run using Terraform, see below section on EFS mounting.
 * Various configurations can be overridden using  `-var=`, see [variables.tf](./terraform/variables.tf) for available parameters.
 See Terraform doc on [variables](https://www.terraform.io/docs/configuration/variables.html) to see how to pass command line arguments.
-* Secure Agent Docker images are tagged using the value, `git-${CI_COMMIT_SHA:0:8}` and push to ECR registry in `us-east-1` in CD pipeline ([.gitlab-ci.yml](.gitlab-ci.yml)). The same 
+* Secure Agent Docker images are tagged using the value, `git-${CI_COMMIT_SHA:0:8}` and push to ECR registry in `us-east-1` in CD pipeline ([.gitlab-ci.yml](.gitlab-ci.yml)). The same
 image can be used for production set up.
 * Note that if this Terraform set up need to use with a different AWS account then the image will have to upload to [AWS ECR](https://aws.amazon.com/ecr/) under that account(so that
 can be passed as a Terraform variable for `image_name`, see below for an example).
-* Once the Agent is deployed it can be seen under `Administrator`->`Runtime Environments` in IICS(the group name can be configured 
+* Once the Agent is deployed it can be seen under `Administrator`->`Runtime Environments` in IICS(the group name can be configured
 in `infaagent.ini` using `InfaAgent.GroupName=aws-interop`).
 * Terraform state files are stored in shared S3 buckets `test-interop-terraform-state` and `prod-interop-terraform-state`.
 
 ### Informatica User
 [user]: #informatica-user
-Credentials for Informatica User (for test and prod instances) are stored in parameter store (in regions `us-east-1` and 
-`us-east-2` respectively) and expected to be available with the following names: 
+Credentials for Informatica User (for test and prod instances) are stored in parameter store (in regions `us-east-1` and
+`us-east-2` respectively) and expected to be available with the following names:
 * username - `/iics/cicd_username`.
-* password - `/iics/cicd_password`. 
+* password - `/iics/cicd_password`.
 
 ### Credentials in Terraform
 [cred]: #credentials-in-terraform
 * It's recommended to define all variables values in a `*.tfvars` file and pass that to terraform using `-var-file` argument.
- This will avoid having any sensitive parameters in bash history. 
+ This will avoid having any sensitive parameters in bash history.
 
 ```shell script
 $ cd terraform
@@ -196,19 +230,19 @@ $ cd terraform
 ```shell script
 $ terraform init -backend-config="bucket=test-interop-terraform-state" \
                  -backend-config="key=ei.secureagent.tfstate" \
-                 -backend-config="region=us-east-1"  
+                 -backend-config="region=us-east-1"
 ```
 #### Production Environment
 [prodenv]: #production-environment
 ```shell script
 $ terraform init -backend-config="bucket=prod-interop-terraform-state" \
                  -backend-config="key=ei.secureagent.tfstate" \
-                 -backend-config="region=us-east-2"  
+                 -backend-config="region=us-east-2"
 ```
 ```shell script
 $ terraform validate
 
-# create `secure_agent.tfvars` with required variables. 
+# create `secure_agent.tfvars` with required variables.
 $ cat secure_agent.tfvars
 image_name="265723766240.dkr.ecr.us-east-1.amazonaws.com/enterprise-integrations/iics_secure_agent:git-8f6f0d24"
 
@@ -235,15 +269,15 @@ $ terraform apply "agent.tfplan"
 
 ### Memory and CPU for Secure Agent
 [memory]: #memory-and-cpu-for-secure-agent
-* By default, Secure Agent is deployed on to a `t2.xlarge` instance and container is given an 15GB of memory. `instance_type` 
+* By default, Secure Agent is deployed on to a `t2.xlarge` instance and container is given an 15GB of memory. `instance_type`
 and `container_memory` variables can be used to adjust these values. See [variables.tf](./terraform/variables.tf) for details.
 
 ### EFS Mounting, Accessing Secure Agent Configurations, Logs and Additional Debugging
 [debug]: #efs-mounting-accessing-secure-agent-configurations-logs-and-additional-debugging
-* Above terraform configuration creates a EFS file system that is used by the container to persist Secure Agent configurations 
+* Above terraform configuration creates a EFS file system that is used by the container to persist Secure Agent configurations
 and logs files.
-* One way to access Secure Agent's log files and configurations files is to mount the EFS file system use by Secure Agent. 
+* One way to access Secure Agent's log files and configurations files is to mount the EFS file system use by Secure Agent.
 EFS file system is accessible from any instance within `test` and `prod` tiers. Start an EC2 instance in `test-tier` VPC
-(or in prod VPC - `prod-tier `) and this instance can be used to mount the EFS file system (be sure to that this EC2 instance can be accessed from SSH). 
+(or in prod VPC - `prod-tier `) and this instance can be used to mount the EFS file system (be sure to that this EC2 instance can be accessed from SSH).
 Then use the instructions documented in `Settings` in Secure Agent's EFS file system(`iics-secure-agent`) under `File system access` in AWS console.
-* If there's a requirement to log into EC2 instance that runs the container, [AWS Session Manager](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager.html) can be [used](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-quick-setup.html). 
+* If there's a requirement to log into EC2 instance that runs the container, [AWS Session Manager](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager.html) can be [used](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-quick-setup.html).

docker-compose.yml

@@ -3,9 +3,29 @@ services:
   iics-secure-agent:
     build: .
     image: iics_secure_agent
-    hostname: "${HOSTNAME}" # Agent name that will show up in the IICS Admin Dashboard
-    stop_grace_period: 5m   # Takes a long time to shutdown gracefully, otherwise Docker will kill it
-    environment:            # Values can be set in a .env file or passed in at runtime
+    # Agent name that will show up in the IICS Admin Dashboard
+    hostname: "${HOSTNAME}"
+    # Takes a long time to shutdown gracefully, otherwise Docker will kill it
+    stop_grace_period: 8m
+    # Values can be set in a .env file or passed in at runtime
+    environment:
       INFORMATICA_USER:
       INFORMATICA_PASSWORD:
       JSON_LOG: "true"
+    volumes:
+      - ${CONFIG_DIR}/odbc.ini:/home/agent/.odbc.ini
+      - ${CONFIG_DIR}/infaagent.ini:${APPS_DIR}/agentcore/conf/infaagent.ini
+      - ${ODBC_DATABASE_DIR}:/home/agent/databases
+      - ${DRIVERS_DIR}:/${APPS_DIR}/Data_Integration_Server/ext/drivers
+      - ${LOG_DIR}/agentcore.log:${APPS_DIR}/agentcore.log
+      - ${LOG_DIR}/infaagent.log:${APPS_DIR}/infaagent.log
+      - ${LOG_DIR}/consoleAgentManager.log:${APPS_DIR}/consoleAgentManager.log
+      - ${LOG_DIR}/agentcore:${APPS_DIR}/agentcore/logs
+      - ${LOG_DIR}/Administrator:${APPS_DIR}/Administrator/logs
+      - ${LOG_DIR}/CIHProcessor:${APPS_DIR}/CIHProcessor/logs
+      - ${LOG_DIR}/Common_Integration_Components:${APPS_DIR}/Common_Integration_Components/logs
+      - ${LOG_DIR}/Data_Integration_Server:${APPS_DIR}/Data_Integration_Server/logs
+      - ${LOG_DIR}/FileIntegrationService:${APPS_DIR}/FileIntegrationService/logs
+      - ${LOG_DIR}/MassIngestionRuntime:${APPS_DIR}/MassIngestionRuntime/logs
+      - ${LOG_DIR}/OpsInsightsDataCollector:${APPS_DIR}/OpsInsightsDataCollector/logs
+      - ${LOG_DIR}/process-engine:${APPS_DIR}/process-engine/logs

odbcinst.ini

+[SQLite3]
+Description = SQLite3 ODBC Driver
+Driver = /usr/lib/x86_64-linux-gnu/odbc/libsqlite3odbc.so
+Setup = /usr/lib/x86_64-linux-gnu/odbc/libsqlite3odbc.so
+UsageCount = 1