for x in mariadb keystone horizon rabbitmq memcached nova-api \
         nova-conductor nova-scheduler glance-api-haproxy \
         glance-registry-haproxy glance-api glance-registry \
         neutron-server neutron-dhcp-agent neutron-l3-agent \
         neutron-metadata-agent neutron-openvswitch-agent \
         openvswitch-db-server openvswitch-vswitchd nova-libvirt \
         nova-compute nova-consoleauth nova-novncproxy \
         nova-novncproxy-haproxy neutron-server-haproxy \
         nova-api-haproxy cinder-api cinder-api-haproxy \
         cinder-backup cinder-scheduler cinder-volume; \
do
    kolla-kubernetes resource create configmap $x
done

for x in mariadb rabbitmq glance; do
    kolla-kubernetes resource create pv $x
    kolla-kubernetes resource create pvc $x
done

#### Had run into issues previously creating the cinder-api. Resolved by enabling cinder in globals.yml

for x in mariadb memcached keystone-admin keystone-public rabbitmq \
         rabbitmq-management nova-api glance-api glance-registry \
         neutron-server nova-metadata nova-novncproxy horizon \
         cinder-api; \
do
    kolla-kubernetes resource create svc $x
done


# mariadb-bootstrap fails sometimes if the mariadb directory is not empty. This can be fixed by deleting the mariadb folder on the server that is running the bootstrap

for x in mariadb-bootstrap rabbitmq-bootstrap; do
    kolla-kubernetes resource create bootstrap $x
done

watch kubectl get jobs --namespace kolla

for x in mariadb-bootstrap rabbitmq-bootstrap; do
    kolla-kubernetes resource delete bootstrap $x
done

for x in mariadb memcached rabbitmq; do
    kolla-kubernetes resource create pod $x
done

watch kubectl get pods --namespace kolla

# The following command seems to fail and restart a few times for keystone endpoints, but is eventually successful after about a minute or so

for x in keystone-create-db keystone-endpoints keystone-manage-db; do
    kolla-kubernetes resource create bootstrap $x
done

watch kubectl get jobs --namespace kolla

for x in keystone-create-db keystone-endpoints keystone-manage-db; do
    kolla-kubernetes resource delete bootstrap $x
done

kolla-kubernetes resource create pod keystone


********** Creation of bootstraps ****************

# We split a for loop into multiple ones below

for x in glance-create-db glance-create-keystone-endpoint-public \
         glance-create-keystone-endpoint-internal glance-create-keystone-endpoint-admin \
         glance-manage-db; \
do
    kolla-kubernetes resource create bootstrap $x
done

# nova-create-api-db takes about 2-3 minutes to be successful

for x in nova-create-api-db nova-create-keystone-endpoint-public \
         nova-create-keystone-endpoint-internal nova-create-keystone-endpoint-admin \
         nova-create-db; \
do
    kolla-kubernetes resource create bootstrap $x
done

# neutron-manage-db takes 2-3 minutes to be successful

for x in neutron-create-db neutron-create-keystone-endpoint-public \
         neutron-create-keystone-endpoint-internal neutron-create-keystone-endpoint-admin \
         neutron-manage-db; \
do
    kolla-kubernetes resource create bootstrap $x
done


for x in cinder-create-db cinder-create-keystone-endpoint-public \
         cinder-create-keystone-endpoint-publicv2 cinder-create-keystone-endpoint-admin \
         cinder-create-keystone-endpoint-adminv2 cinder-create-keystone-endpoint-internal \
         cinder-create-keystone-endpoint-internalv2 cinder-manage-db; \
do
    kolla-kubernetes resource create bootstrap $x
done


******* Deleting of bootstraps *************

# We split a for loop into multiple ones below

for x in glance-create-db glance-create-keystone-endpoint-public \
         glance-create-keystone-endpoint-internal glance-create-keystone-endpoint-admin \
         glance-manage-db; \
do
    kolla-kubernetes resource delete bootstrap $x
done

for x in nova-create-api-db nova-create-keystone-endpoint-public \
         nova-create-keystone-endpoint-internal nova-create-keystone-endpoint-admin \
         nova-create-db; \
do
    kolla-kubernetes resource delete bootstrap $x
done


for x in neutron-create-db neutron-create-keystone-endpoint-public \
         neutron-create-keystone-endpoint-internal neutron-create-keystone-endpoint-admin \
         neutron-manage-db; \
do
    kolla-kubernetes resource delete bootstrap $x
done

for x in cinder-create-db cinder-create-keystone-endpoint-public \
         cinder-create-keystone-endpoint-publicv2 cinder-create-keystone-endpoint-admin \
         cinder-create-keystone-endpoint-adminv2 cinder-create-keystone-endpoint-internal \
         cinder-create-keystone-endpoint-internalv2 cinder-manage-db; \
do
    kolla-kubernetes resource delete bootstrap $x
done


****************** Creation of pods ***********

#updated nova-control-conductor-pod.yml.j2 to change from PetSet to StatefulSet
#updated nova-control-scheduler-pod.yml.j2 to change from PetSet to StatefulSet
#updated nova-control-consoleauth-pod.yml.j2 to change from PetSet to StatefulSet

for x in nova-api nova-conductor nova-scheduler glance-api \
         glance-registry neutron-server horizon nova-consoleauth \
         nova-novncproxy cinder-api cinder-scheduler; \
do
    kolla-kubernetes resource create pod $x
done


************* Creation of other pods ************

# Updated openvswitch-ovsdb-daemonset.yml.j2 to add imagePullPolicy
# Updated openvswitch-vswitchb-daemonset.yml.j2 to add imagePullPolicy
# Updated neutron-openvswitch-agent-daemonset.yml.j2 to add imagePullPolicy in two places
# Updated neutron-l3-agent-daemonset.yml.j2 to add imagePullPolicy in two places
# "imagePullPolicy": "IfNotPresent",


for x in openvswitch-ovsdb-network openvswitch-vswitchd-network \
         neutron-openvswitch-agent-network neutron-dhcp-agent \
         neutron-metadata-agent-network neutron-l3-agent-network; \
do
    kolla-kubernetes resource create pod $x
done


# Updated the following files to add imagePullPolicy
# nova-libvirt-pod.yml.j2
# nova-compute-pod.yml.j2 in two places

kolla-kubernetes resource create pod nova-libvirt
kolla-kubernetes resource create pod nova-compute