Forum Discussion

leo_uribe's avatar
leo_uribe
Icon for Nimbostratus rankNimbostratus
May 02, 2023

f5-declarative-onboarding error 422

Recieving the following error when trying to deploy an AWS BIGIP instance using a Terraform templatefile() function to pass variables to cloud-init template file, 

tmos_do_only_wait - DEBUG - f5-declarative-onboarding error 422 - {u'message': u'request failed with null exception', u'code': 422, u'referer': u'Unknown', u'restOperationId': 6674184, u'kind': u':resterrorresponse'}

This errors occurs after the icontrollx packages (DO, AS3) are installed. When I use static values along with the file() function in the user_data argument the BIGIP has no issues accepting the configuration. I've verified my configs and compared it to examples provided in github.com/F5Networks/terraform-aws-bigip-module with no success. 

Code below

 

 

~variables.tf~
variable "name_servers" { default = "" }
variable "ltm_eth0_sg_id" { default = "" }
variable "ltm_eth0_subnet_id" { default = "" }
variable "ltm_eth0_subnet_cidr_block" { default = "" }
variable "ltm_eth1_sg_id" { default = "" }
variable "ltm_eth1_subnet_id" { default = "" }
variable "ltm_eth1_subnet_cidr_block" { default = "" }
variable "ltm_eth2_sg_id" { default = "" }
variable "ltm_eth2_subnet_id" { default = "" }
variable "ltm_eth2_subnet_cidr_block" { default = "" }
variable "ltm_eth3_sg_id" { default = "" }
variable "ltm_eth3_subnet_id" { default = "" }
variable "ltm_eth3_subnet_cidr_block" { default = "" }
variable "ltm_eth4_sg_id" { default = "" }
variable "ltm_eth4_subnet_id" { default = "" }
variable "ltm_eth4_subnet_cidr_block" { default = "" }
variable "ltm_eth5_sg_id" { default = "" }
variable "ltm_eth5_subnet_id" { default = "" }
variable "ltm_eth5_subnet_cidr_block" { default = "" }
variable "ltm_eth6_sg_id" { default = "" }
variable "ltm_eth6_subnet_id" { default = "" }
variable "ltm_eth6_subnet_cidr_block" { default = "" }
variable "ltm_eth7_sg_id" { default = "" }
variable "ltm_eth7_subnet_id" { default = "" }
variable "ltm_eth7_subnet_cidr_block" { default = "" }
variable "ltm2_eth0_intf_ip" { default = "" }
variable "ltm2_eth1_intf_ip" { default = "" }
variable "ltm2_eth2_intf_ip" { default = "" }
variable "ltm2_eth3_intf_ip" { default = "" }
variable "ltm2_eth4_intf_ip" { default = "" }
variable "ltm2_eth5_intf_ip" { default = "" }
variable "ltm2_eth6_intf_ip" { default = "" }
variable "ltm2_eth7_intf_ip" { default = "" }

~main.tf~
  module "ltm" {
  name_servers                   = ["169.254.169.253", "8.8.8.8"]
  ltm_eth0_sg_id                 = module.security_groups.ltm_mgmt_security_group_id
  ltm_eth0_subnet_id             = module.subnet.ltm_gtm_mgmt_subnet_id
  ltm_eth0_subnet_cidr_block     = module.subnet.ltm_gtm_mgmt_subnet_cidr_block
  ltm_eth1_sg_id                 = module.security_groups.ltm_routing_security_group_id
  ltm_eth1_subnet_id             = module.subnet.ltm_gtm_routing_subnet_id
  ltm_eth1_subnet_cidr_block     = module.subnet.ltm_gtm_routing_subnet_cidr_block
  ltm_eth2_sg_id                 = module.security_groups.ltm_external_security_group_id
  ltm_eth2_subnet_id             = module.subnet.ltm_ext_vip1_subnet_id
  ltm_eth2_subnet_cidr_block     = module.subnet.ltm_ext_vip1_subnet_cidr_block
  ltm_eth3_sg_id                 = module.security_groups.ltm_external_security_group_id
  ltm_eth3_subnet_id             = module.subnet.ltm_ext_vip2_subnet_id
  ltm_eth3_subnet_cidr_block     = module.subnet.ltm_ext_vip2_subnet_cidr_block
  ltm_eth4_sg_id                 = module.security_groups.ltm_external_security_group_id
  ltm_eth4_subnet_id             = module.subnet.ltm_ext_vip3_subnet_id
  ltm_eth4_subnet_cidr_block     = module.subnet.ltm_ext_vip3_subnet_cidr_block
  ltm_eth5_sg_id                 = module.security_groups.ltm_internal_security_group_id
  ltm_eth5_subnet_id             = module.subnet.ltm_int_vip1_subnet_id
  ltm_eth5_subnet_cidr_block     = module.subnet.ltm_int_vip1_subnet_cidr_block
  ltm_eth6_sg_id                 = module.security_groups.ltm_internal_security_group_id
  ltm_eth6_subnet_id             = module.subnet.ltm_int_vip2_subnet_id
  ltm_eth6_subnet_cidr_block     = module.subnet.ltm_int_vip2_subnet_cidr_block
  ltm_eth7_sg_id                 = module.security_groups.ltm_internal_security_group_id
  ltm_eth7_subnet_id             = module.subnet.ltm_int_vip3_subnet_id
  ltm_eth7_subnet_cidr_block     = module.subnet.ltm_int_vip3_subnet_cidr_block
  ltm2_eth0_intf_ip              = ["10.169.96.7"]
  ltm2_eth1_intf_ip              = ["10.169.96.72"]
  ltm2_eth2_intf_ip              = ["10.169.97.40"]
  ltm2_eth3_intf_ip              = ["10.169.98.40"]
  ltm2_eth4_intf_ip              = ["10.169.99.40"]
  ltm2_eth5_intf_ip              = ["10.169.100.40"]
  ltm2_eth6_intf_ip              = ["10.169.101.40"]
  ltm2_eth7_intf_ip              = ["10.169.102.40"]
  }

~ltm2_main.tf~
resource "aws_network_interface" "ltm2_eth0_intf" {
  subnet_id         = var.ltm_eth0_subnet_id
  private_ips       = var.ltm2_eth0_intf_ip
  security_groups   = [var.ltm_eth0_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH0"
  }
}
resource "aws_network_interface" "ltm2_eth1_intf" {
  subnet_id         = var.ltm_eth1_subnet_id
  private_ips       = var.ltm2_eth1_intf_ip
  security_groups   = [var.ltm_eth1_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH1"
  }
}
resource "aws_network_interface" "ltm2_eth2_intf" {
  subnet_id         = var.ltm_eth2_subnet_id
  private_ips       = var.ltm2_eth2_intf_ip
  security_groups   = [var.ltm_eth2_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH2"
  }
}

resource "aws_network_interface" "ltm2_eth3_intf" {
  subnet_id         = var.ltm_eth3_subnet_id
  private_ips       = var.ltm2_eth3_intf_ip
  security_groups   = [var.ltm_eth3_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH3"
  }
}
resource "aws_network_interface" "ltm2_eth4_intf" {
  subnet_id         = var.ltm_eth4_subnet_id
  private_ips       = var.ltm2_eth4_intf_ip
  security_groups   = [var.ltm_eth4_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH4"
  }
}
resource "aws_network_interface" "ltm2_eth5_intf" {
  subnet_id         = var.ltm_eth5_subnet_id
  private_ips       = var.ltm2_eth5_intf_ip
  security_groups   = [var.ltm_eth5_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH5"
  }
}
resource "aws_network_interface" "ltm2_eth6_intf" {
  subnet_id         = var.ltm_eth6_subnet_id
  private_ips       = var.ltm2_eth6_intf_ip
  security_groups   = [var.ltm_eth6_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH6"
  }
}
resource "aws_network_interface" "ltm2_eth7_intf" {
  subnet_id         = var.ltm_eth7_subnet_id
  private_ips       = var.ltm2_eth7_intf_ip
  security_groups   = [var.ltm_eth7_sg_id]
  source_dest_check = "true"
  tags = {
    Name = "LTM2:ETH7"
  }
}
resource "aws_instance" "ltm2" {
  ami           = "ami-07cda73ac7991dfc8"    
  instance_type = "c4.8xlarge"
  key_name      = "ltm2-key"
  disable_api_termination = "false"
  ebs_optimized           = "false"
  root_block_device {
    #device_name = "/dev/xvda"
    volume_type = "standard"
    volume_size = "142"
  }
  ebs_block_device {
    device_name = "/dev/xvdb"
    volume_type = "standard"
    volume_size = "20"
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth0_intf[0].id
    device_index         = 0
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth1_intf[0].id
    device_index         = 1
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth2_intf[0].id
    device_index         = 2
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth3_intf[0].id
    device_index         = 3
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth4_intf[0].id
    device_index         = 4
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth5_intf[0].id
    device_index         = 5
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth6_intf[0].id
    device_index         = 6
  }
  network_interface {
    network_interface_id = aws_network_interface.ltm2_eth7_intf[0].id
    device_index         = 7
  }
  user_data = templatefile("${path.module}/ltm.tpl", {
    hostname         = "ltm2-lab"
    name_server1     = var.name_servers[0]
    name_server2     = var.name_servers[1]
    ltm_ext_vips1_ip = var.ltm2_eth2_intf_ip[0]
    ltm_ext_vips2_ip = var.ltm2_eth3_intf_ip[0]
    ltm_ext_vips3_ip = var.ltm2_eth4_intf_ip[0]
    ltm_int_vips1_ip = var.ltm2_eth5_intf_ip[0]
    ltm_int_vips2_ip = var.ltm2_eth6_intf_ip[0]
    ltm_int_vips3_ip = var.ltm2_eth7_intf_ip[0]
    ltm_traffic_ip   = var.ltm2_eth1_intf_ip[0]
    admin_pwd        = "Password1234"
  })

  tags = {
    Name = "ltm2-lab"
  }
}

~ltm.tpl~
#cloud-config
tmos_declared:
  enabled: true
  icontrollx_trusted_sources: false
  icontrollx_package_urls:
    - "https://github.com/F5Networks/f5-declarative-onboarding/releases/download/v1.37.0/f5-declarative-onboarding-1.37.0-3.noarch.rpm"
    - "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.44.0/f5-appsvcs-3.44.0-3.noarch.rpm"
  do_declaration:
    schemaVersion: 1.0.0
    class: Device
    async: true
    label: Cloudinit Onboarding
    Common:
      class: Tenant
      provisioningLevels:
        class: Provision
        ltm: dedicated
      System:
        class: System
        hostname: ${hostname}
        mgmtDhcpEnabled: false
        autoCheck: false
        autoPhonehome: true
      dnsServers:
        class: DNS
        nameServers:
          - ${name_server1}
          - ${name_server2}
        search:
          - ops.sfdc.net
      ntpServers:
        class: NTP
        servers:
          - ${name_server1}
          - ${name_server2}
      EXT-VIPS-1-VLAN:
        class: VLAN
        tag: 102
        interfaces:
          - name: "1.2"
            tagged: false
      SelfIp_EXT-VIPS-1:
        class: SelfIp
        address: ${ltm_ext_vips1_ip}/24
        vlan: EXT-VIPS-1-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      EXT-VIPS-2-VLAN:
        class: VLAN
        tag: 103
        interfaces:
          - name: "1.3"
            tagged: false
      SelfIp_EXT-VIPS-2:
        class: SelfIp
        address: ${ltm_ext_vips2_ip}/24
        vlan: EXT-VIPS-2-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      EXT-VIPS-3-VLAN:
        class: VLAN
        tag: 104
        interfaces:
          - name: "1.4"
            tagged: false
      SelfIp_EXT-VIPS-3:
        class: SelfIp
        address: ${ltm_ext_vips3_ip}/24
        vlan: EXT-VIPS-3-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      INT-VIPS-1-VLAN:
        class: VLAN
        tag: 105
        interfaces:
          - name: "1.5"
            tagged: false
      SelfIp_INT-VIPS-1:
        class: SelfIp
        address: ${ltm_int_vips1_ip}/24
        vlan: INT-VIPS-1-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      INT-VIPS-2-VLAN:
        class: VLAN
        tag: 106
        interfaces:
          - name: "1.6"
            tagged: false
      SelfIp_INT-VIPS-2:
        class: SelfIp
        address: ${ltm_int_vips2_ip}/24
        vlan: INT-VIPS-2-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      INT-VIPS-3-VLAN:
        class: VLAN
        tag: 107
        interfaces:
          - name: "1.7"
            tagged: false
      SelfIp_INT-VIPS-3:
        class: SelfIp
        address: ${ltm_int_vips3_ip}/24
        vlan: INT-VIPS-3-VLAN
        trafficGroup: traffic-group-local-only
        allowService: none
      TRAFFIC-VLAN:
        class: VLAN
        tag: 101
        interfaces:
          - name: "1.1"
            tagged: false
      SelfIp_TRAFFIC:
        class: SelfIp
        address: ${ltm_traffic_ip}/27
        vlan: TRAFFIC-VLAN
        trafficGroup: traffic-group-local-only
        allowService: default
chpasswd:
  list: |
    admin:${admin_pwd}
  expire: False

 

 

 

 

 

  • Maybe then better contact support as I see a lot of 422 errors but for AS3 not DO, so it could be something new.

  • This could be terraform issue as you said that in some cases you push the DO.

    Have you tried ansible?

     

    https://clouddocs.f5.com/products/orchestration/ansible/devel/f5_bigip/modules_2_0/bigip_do_deploy_module.html#bigip-do-deploy-module-2

     

    You can replace the file lookup with template lookup and the template can have variables and not be static as a file.

     

    https://docs.ansible.com/ansible/latest/collections/ansible/builtin/template_lookup.html

     

     

    Also shouldn't templatefile()  souldn't it have a destination file option that will be created and then you can use the destination file?

     

    https://www.cloudbolt.io/terraform-best-practices/terraform-template/

     

    Maybe also test depends-on option as the terraform resources to depend on the datasource as to be triggered after it.

     

    https://developer.hashicorp.com/terraform/language/meta-arguments/depends_on

    • leo_uribe's avatar
      leo_uribe
      Icon for Nimbostratus rankNimbostratus

      We would like to stick with Terraform as we plan on managing all of the app services using Terraform aswell. The odd part is the rendered output (below) from Terraform looks to be accurate. The destination file isn't required since we are passing this to AWS cloud-init agent via the "user_data" argument. 

      #cloud-config
      tmos_declared:
        enabled: true
        icontrollx_trusted_sources: false
        icontrollx_package_urls:
          - "https://github.com/F5Networks/f5-declarative-onboarding/releases/download/v1.37.0/f5-declarative-onboarding-1.37.0-3.noarch.rpm"
          - "https://github.com/F5Networks/f5-appsvcs-extension/releases/download/v3.44.0/f5-appsvcs-3.44.0-3.noarch.rpm"
        do_declaration:
          schemaVersion: 1.0.0
          class: Device
          async: true
          label: Cloudinit Onboarding
          Common:
            class: Tenant
            provisioningLevels:
              class: Provision
              ltm: dedicated
            System:
              class: System
              hostname: dpl2-labnetsec
              mgmtDhcpEnabled: false
              autoCheck: false
              autoPhonehome: true
            dnsServers:
              class: DNS
              nameServers:
                - 169.254.169.253
                - 8.8.8.8
              search:
                - ops.sfdc.net
            ntpServers:
              class: NTP
              servers:
                - 169.254.169.253
                - 8.8.8.8
            EXT-VIPS-1-VLAN:
              class: VLAN
              tag: 102
              interfaces:
                - name: "1.2"
                  tagged: false
            SelfIp_EXT-VIPS-1:
              class: SelfIp
              address: 10.169.97.40/24
              vlan: EXT-VIPS-1-VLAN
              trafficGroup: traffic-group-local-only
              allowService: none
            EXT-VIPS-2-VLAN:
              class: VLAN
              tag: 103
              interfaces:
                - name: "1.3"
                  tagged: false
            SelfIp_EXT-VIPS-2:
              class: SelfIp
              address: 10.169.98.40/24
              vlan: EXT-VIPS-2-VLAN
              trafficGroup: traffic-group-local-only
              allowService: none
            EXT-VIPS-3-VLAN:
              class: VLAN
              tag: 104
              interfaces:
                - name: "1.4"
                  tagged: false
            SelfIp_EXT-VIPS-3:
              class: SelfIp
              address: 10.169.99.40/24
              vlan: EXT-VIPS-3-VLAN
              trafficGroup: traffic-group-local-only
              allowService: none
            INT-VIPS-1-VLAN:
              class: VLAN
              tag: 105
              interfaces:
                - name: "1.5"
                  tagged: false
            SelfIp_INT-VIPS-1:
              class: SelfIp
              address: 10.169.100.40/24
              vlan: INT-VIPS-1-VLAN
              trafficGroup: traffic-group-local-only
              allowService: none
            INT-VIPS-2-VLAN:
              class: VLAN
              tag: 106
              interfaces:
                - name: "1.6"
                  tagged: false
            SelfIp_INT-VIPS-2:
              class: SelfIp
              address: 10.169.101.40/24
              vlan: INT-VIPS-2-VLAN
              trafficGroup: traffic-group-local-only
              allowService: none
            INT-VIPS-3-VLAN:
              class: VLAN
              tag: 107
              interfaces:
                - name: "1.7"
                  tagged: false
            SelfIp_INT-VIPS-3:
              class: SelfIp
              address: 10.169.102.40/24
              vlan: INT-VIPS-3-VLAN
              trafficGroup: traffic-group-local-only
              allowService: none
            TRAFFIC-VLAN:
              class: VLAN
              tag: 101
              interfaces:
                - name: "1.1"
                  tagged: false
            SelfIp_TRAFFIC:
              class: SelfIp
              address: 10.169.96.72/27
              vlan: TRAFFIC-VLAN
              trafficGroup: traffic-group-local-only
              allowService: default
      chpasswd:
        list: |
          admin:eni-abcd1234
        expire: False

       

      • Nikoolayy1's avatar
        Nikoolayy1
        Icon for MVP rankMVP

        Maybe then better contact support as I see a lot of 422 errors but for AS3 not DO, so it could be something new.