From 428018cbe505101d6f034fa4a0aaf53fd8f2caf1 Mon Sep 17 00:00:00 2001
From: Tlacenka <tlacencin@gmail.com>
Date: Fri, 13 Oct 2017 16:42:07 +0200
Subject: Add Extra Docker Registry URLs (custom post-provision action) (#794)

* add-docker-registry: playbook that adds docker registries to docker config file (in progress)

* indentation fix

* docker registries: add check for variable type

* another type conversion

* docker registry: try another unified formatting

* another attempt

* type error fix

* quotation attempt

* docker registry: bug fixes

* docker registry: fixed formatting

* docker registry: if docker is not available, skip the whole playbook

* README updated

* README: typo

* docker registries: suggested changes applied (in progress)

* docker registries: README updated, redundant check removed

* removed redundant become:true
---
 playbooks/provisioning/openstack/README.md         | 19 +++++
 .../custom-actions/add-docker-registry.yml         | 90 ++++++++++++++++++++++
 2 files changed, 109 insertions(+)
 create mode 100644 playbooks/provisioning/openstack/custom-actions/add-docker-registry.yml

diff --git a/playbooks/provisioning/openstack/README.md b/playbooks/provisioning/openstack/README.md
index 370f582b2..78d4ffe7c 100644
--- a/playbooks/provisioning/openstack/README.md
+++ b/playbooks/provisioning/openstack/README.md
@@ -584,12 +584,31 @@ This playbook runs against all cluster nodes. In order to help prevent slow conn
 problems, the task is retried 10 times in case of initial failure.
 Note that in order for this example to work in your deployment, your servers must use the RHEL image.
 
+#### Adding extra Docker registry URLs
+
+This playbook is located in the [custom-actions](https://github.com/openshift/openshift-ansible-contrib/tree/master/playbooks/provisioning/openstack/custom-actions) directory.
+
+It adds URLs passed as arguments to the docker configuration program.
+Going into more detail, the configuration program (which is in the YAML format) is loaded into an ansible variable
+([lines 27-30](https://github.com/openshift/openshift-ansible-contrib/blob/master/playbooks/provisioning/openstack/custom-actions/add-docker-registry.yml#L27-L30))
+and in its structure, `registries` and `insecure_registries` sections are expanded with the newly added items
+([lines 56-76](https://github.com/openshift/openshift-ansible-contrib/blob/master/playbooks/provisioning/openstack/custom-actions/add-docker-registry.yml#L56-L76)).
+The new content is then saved into the original file
+([lines 78-82](https://github.com/openshift/openshift-ansible-contrib/blob/master/playbooks/provisioning/openstack/custom-actions/add-docker-registry.yml#L78-L82))
+and docker is restarted.
+
+Example usage:
+```
+ansible-playbook -i <inventory> openshift-ansible-contrib/playbooks/provisioning/openstack/custom-actions/add-docker-registry.yml  --extra-vars '{"registries": "reg1", "insecure_registries": ["ins_reg1","ins_reg2"]}'
+```
+
 Please consider contributing your custom playbook back to openshift-ansible-contrib!
 
 A library of custom post-provision actions exists in `openshift-ansible-contrib/playbooks/provisioning/openstack/custom-actions`. Playbooks include:
 
 * [add-yum-repos.yml](https://github.com/openshift/openshift-ansible-contrib/blob/master/playbooks/provisioning/openstack/custom-actions/add-yum-repos.yml): adds a list of custom yum repositories to every node in the cluster
 * [add-rhn-pools.yml](https://github.com/openshift/openshift-ansible-contrib/blob/master/playbooks/provisioning/openstack/custom-actions/add-rhn-pools.yml): attaches a list of additional RHN pools to every node in the cluster
+* [add-docker-registry.yml](https://github.com/openshift/openshift-ansible-contrib/blob/master/playbooks/provisioning/openstack/custom-actions/add-docker-registry.yml): adds a list of docker registries to the docker configuration on every node in the cluster
 
 ### Install OpenShift
 
diff --git a/playbooks/provisioning/openstack/custom-actions/add-docker-registry.yml b/playbooks/provisioning/openstack/custom-actions/add-docker-registry.yml
new file mode 100644
index 000000000..e118a71dc
--- /dev/null
+++ b/playbooks/provisioning/openstack/custom-actions/add-docker-registry.yml
@@ -0,0 +1,90 @@
+---
+- hosts: OSEv3
+  become: true
+  vars:
+    registries: []
+    insecure_registries: []
+
+  tasks:
+  - name: Check if docker is even installed
+    command: docker
+
+  - name: Install atomic-registries package
+    yum:
+      name: atomic-registries
+      state: latest
+
+  - name: Get registry configuration file
+    register: file_result
+    stat:
+      path: /etc/containers/registries.conf
+
+  - name: Check if it exists
+    assert:
+      that: 'file_result.stat.exists'
+      msg: "Configuration file does not exist."
+
+  - name: Load configuration file
+    shell: cat /etc/containers/registries.conf
+    register: file_content
+
+  - name: Store file content into a variable
+    set_fact:
+      docker_conf: "{{ file_content.stdout | from_yaml }}"
+
+  - name: Make sure that docker file content is a dictionary
+    when: '(docker_conf is string) and (not docker_conf)'
+    set_fact:
+      docker_conf: {}
+
+  - name: Make sure that registries is a list
+    when: 'registries is string'
+    set_fact:
+      registries_list: [ "{{ registries }}" ]
+
+  - name: Make sure that insecure_registries is a list
+    when: 'insecure_registries is string'
+    set_fact:
+      insecure_registries_list: [ "{{ insecure_registries }}" ]
+
+  - name: Set default values if there are no registries defined
+    set_fact:
+      docker_conf_registries: "{{ [] if docker_conf['registries'] is not defined else docker_conf['registries'] }}"
+      docker_conf_insecure_registries: "{{ [] if docker_conf['insecure_registries'] is not defined else docker_conf['insecure_registries'] }}"
+
+  - name: Add other registries
+    when: 'registries_list is not defined'
+    register: registries_merge_result
+    set_fact:
+      docker_conf: "{{ docker_conf | combine({'registries': (docker_conf_registries + registries) | unique}, recursive=True) }}"
+
+  - name: Add other registries (if registries had to be converted)
+    when: 'registries_merge_result|skipped'
+    set_fact:
+      docker_conf: "{{ docker_conf | combine({'registries': (docker_conf_registries + registries_list) | unique}, recursive=True) }}"
+
+  - name: Add insecure registries
+    when: 'insecure_registries_list is not defined'
+    register: insecure_registries_merge_result
+    set_fact:
+      docker_conf: "{{ docker_conf | combine({'insecure_registries': (docker_conf_insecure_registries + insecure_registries) | unique }, recursive=True) }}"
+
+  - name: Add insecure registries (if insecure_registries had to be converted)
+    when: 'insecure_registries_merge_result|skipped'
+    set_fact:
+      docker_conf: "{{ docker_conf | combine({'insecure_registries': (docker_conf_insecure_registries + insecure_registries_list) | unique }, recursive=True) }}"
+
+  - name: Load variable back to file
+    copy:
+      content: "{{ docker_conf | to_yaml }}"
+      dest: /etc/containers/registries.conf
+
+  - name: Restart registries service
+    service:
+      name: registries
+      state: restarted
+
+  - name: Restart docker
+    service:
+      name: docker
+      state: restarted
-- 
cgit v1.2.3

-- 
cgit v1.2.3

-- 
cgit v1.2.3

-- 
cgit v1.2.3

-- 
cgit v1.2.3

-- 
cgit v1.2.3

-- 
cgit v1.2.3

-- 
cgit v1.2.3

-- 
cgit v1.2.3

-- 
cgit v1.2.3

-- 
cgit v1.2.3

-- 
cgit v1.2.3

-- 
cgit v1.2.3

-- 
cgit v1.2.3

-- 
cgit v1.2.3


From ca88364175fe5177cecbb479a157d7329db05d8a Mon Sep 17 00:00:00 2001
From: Bogdan Dobrelya <bdobreli@redhat.com>
Date: Mon, 16 Oct 2017 15:42:42 +0200
Subject: Support separate data network for Flannel SDN (#757)

* Support separate data network for Flannel SDN

Document the use case for a separate flannel data network.
Allow Nova servers for openshift cluster to be provisioned
with that isolated data network created and connected to
masters, computes and infra nodes. Do not configure dns
nameservers and router for that network.

Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>

* Fix flannel use cases with provider network

Provider network cannot be used with flannel SDN
as the latter requires a separate isolated network,
while the provider network is an externally managed
single network.

Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>

* Drop unused data_net_name

Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>
---
 playbooks/provisioning/openstack/README.md         |  9 +++-
 .../provisioning/openstack/net_vars_check.yaml     | 14 ++++++
 playbooks/provisioning/openstack/prerequisites.yml |  3 ++
 .../sample-inventory/group_vars/OSEv3.yml          |  4 ++
 .../openstack/sample-inventory/group_vars/all.yml  |  4 ++
 roles/openstack-stack/templates/heat_stack.yaml.j2 | 31 ++++++++++++
 .../templates/heat_stack_server.yaml.j2            | 57 ++++++++++++++++++++++
 .../templates/heat_stack_server_nofloating.yaml.j2 | 55 +++++++++++++++++++++
 8 files changed, 176 insertions(+), 1 deletion(-)
 create mode 100644 playbooks/provisioning/openstack/net_vars_check.yaml

diff --git a/playbooks/provisioning/openstack/README.md b/playbooks/provisioning/openstack/README.md
index 78d4ffe7c..b9a3b23de 100644
--- a/playbooks/provisioning/openstack/README.md
+++ b/playbooks/provisioning/openstack/README.md
@@ -250,6 +250,9 @@ right after provisioning will fail (unless you're using an external DNS server
 your provider network knows about). You must make sure your nodes are able to
 resolve each other by name.
 
+**NOTE**: Flannel SDN requires a dedicated containers data network and cannot
+work over a single provider network.
+
 #### Security notes
 
 Configure required `*_ingress_cidr` variables to restrict public access
@@ -267,6 +270,10 @@ be the case for development environments. When turned off, the servers will
 be provisioned omitting the ``yum update`` command. This brings security
 implications though, and is not recommended for production deployments.
 
+Flannel network used for user applications and workloads data should be
+isolated from other networks as it has Neutron ports security disabled.
+Openshift master, compute and infra nodes will be connected to that network.
+
 ##### DNS servers security options
 
 Aside from `node_ingress_cidr` restricting public access to in-stack DNS
@@ -646,7 +653,7 @@ The `increment_by` variable is used to specify by how much the deployment should
 be scaled up (if none exists, it serves as a target number of application nodes).
 The path to `openshift-ansible` directory can be customised by the `openshift_ansible_dir`
 variable. Its value must be an absolute path to `openshift-ansible` and it cannot
-contain the '/' symbol at the end. 
+contain the '/' symbol at the end.
 
 Usage:
 
diff --git a/playbooks/provisioning/openstack/net_vars_check.yaml b/playbooks/provisioning/openstack/net_vars_check.yaml
new file mode 100644
index 000000000..68afde415
--- /dev/null
+++ b/playbooks/provisioning/openstack/net_vars_check.yaml
@@ -0,0 +1,14 @@
+---
+- name: Check the provider network configuration
+  fail:
+    msg: "Flannel SDN requires a dedicated containers data network and can not work over a provider network"
+  when:
+    - openstack_provider_network_name is defined
+    - openstack_private_data_network_name is defined
+
+- name: Check the flannel network configuration
+  fail:
+    msg: "A dedicated containers data network is only supported with Flannel SDN"
+  when:
+    - openstack_private_data_network_name is defined
+    - not openshift_use_flannel|default(False)|bool
diff --git a/playbooks/provisioning/openstack/prerequisites.yml b/playbooks/provisioning/openstack/prerequisites.yml
index f2f720f8b..11a31411e 100644
--- a/playbooks/provisioning/openstack/prerequisites.yml
+++ b/playbooks/provisioning/openstack/prerequisites.yml
@@ -2,6 +2,9 @@
 - hosts: localhost
   tasks:
 
+  # Sanity check of inventory variables
+  - include: net_vars_check.yaml
+
   # Check ansible
   - name: Check Ansible version
     assert:
diff --git a/playbooks/provisioning/openstack/sample-inventory/group_vars/OSEv3.yml b/playbooks/provisioning/openstack/sample-inventory/group_vars/OSEv3.yml
index 2e897102e..70e77662d 100644
--- a/playbooks/provisioning/openstack/sample-inventory/group_vars/OSEv3.yml
+++ b/playbooks/provisioning/openstack/sample-inventory/group_vars/OSEv3.yml
@@ -51,3 +51,7 @@ openshift_override_hostname_check: true
 # NOTE(shadower): Always switch to root on the OSEv3 nodes.
 # openshift-ansible requires an explicit `become`.
 ansible_become: true
+
+# # Flannel networking
+#openshift_use_openshift_sdn: false
+#openshift_use_flannel: true
diff --git a/playbooks/provisioning/openstack/sample-inventory/group_vars/all.yml b/playbooks/provisioning/openstack/sample-inventory/group_vars/all.yml
index fa1fb6c64..83289307d 100644
--- a/playbooks/provisioning/openstack/sample-inventory/group_vars/all.yml
+++ b/playbooks/provisioning/openstack/sample-inventory/group_vars/all.yml
@@ -15,6 +15,10 @@ public_dns_nameservers: []
 openstack_ssh_public_key: "openshift"
 openstack_external_network_name: "public"
 #openstack_private_network_name:  "openshift-ansible-{{ stack_name }}-net"
+# # A dedicated Neutron network name for containers data network
+# # Configures the data network to be separated from openstack_private_network_name
+# # NOTE: this is only supported with Flannel SDN yet
+#openstack_private_data_network_name: "openshift-ansible-{{ stack_name }}-data-net"
 
 ## If you want to use a provider network, set its name here.
 ## NOTE: the `openstack_external_network_name` and
diff --git a/roles/openstack-stack/templates/heat_stack.yaml.j2 b/roles/openstack-stack/templates/heat_stack.yaml.j2
index a6b088efb..1f1e33cf2 100644
--- a/roles/openstack-stack/templates/heat_stack.yaml.j2
+++ b/roles/openstack-stack/templates/heat_stack.yaml.j2
@@ -113,6 +113,22 @@ resources:
         - {{ nameserver }}
 {% endfor %}
 
+{% if openshift_use_flannel|default(False)|bool %}
+  data_net:
+    type: OS::Neutron::Net
+    properties:
+      name: openshift-ansible-{{ stack_name }}-data-net
+      port_security_enabled: false
+
+  data_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      name: openshift-ansible-{{ stack_name }}-data-subnet
+      network: { get_resource: data_net }
+      cidr: {{ osm_cluster_network_cidr|default('10.128.0.0/14') }}
+      gateway_ip: null
+{% endif %}
+
   router:
     type: OS::Neutron::Router
     properties:
@@ -641,6 +657,11 @@ resources:
               template: openshift-ansible-cluster_id-net
               params:
                 cluster_id: {{ stack_name }}
+{% if openshift_use_flannel|default(False)|bool %}
+          attach_data_net: true
+          data_net:    { get_resource: data_net }
+          data_subnet: { get_resource: data_subnet }
+{% endif %}
 {% endif %}
           secgrp:
 {% if openstack_flat_secgrp|default(False)|bool %}
@@ -713,6 +734,11 @@ resources:
               template: openshift-ansible-cluster_id-net
               params:
                 cluster_id: {{ stack_name }}
+{% if openshift_use_flannel|default(False)|bool %}
+          attach_data_net: true
+          data_net:    { get_resource: data_net }
+          data_subnet: { get_resource: data_subnet }
+{% endif %}
 {% endif %}
           secgrp:
             - { get_resource: {% if openstack_flat_secgrp|default(False)|bool %}flat-secgrp{% else %}node-secgrp{% endif %} }
@@ -767,6 +793,11 @@ resources:
               template: openshift-ansible-cluster_id-net
               params:
                 cluster_id: {{ stack_name }}
+{% if openshift_use_flannel|default(False)|bool %}
+          attach_data_net: true
+          data_net:    { get_resource: data_net }
+          data_subnet: { get_resource: data_subnet }
+{% endif %}
 {% endif %}
           secgrp:
 # TODO(bogdando) filter only required node rules into infra-secgrp
diff --git a/roles/openstack-stack/templates/heat_stack_server.yaml.j2 b/roles/openstack-stack/templates/heat_stack_server.yaml.j2
index 66c2491a9..6552e0a0d 100644
--- a/roles/openstack-stack/templates/heat_stack_server.yaml.j2
+++ b/roles/openstack-stack/templates/heat_stack_server.yaml.j2
@@ -68,6 +68,28 @@ parameters:
     description: Subnet resource
 {% endif %}
 
+{% if openshift_use_flannel|default(False)|bool %}
+  attach_data_net:
+    type: boolean
+    default: false
+    label: Attach-data-net
+    description: A switch for data port connection
+
+  data_net:
+    type: string
+    default: ''
+    label: Net ID
+    description: Net resource
+
+{% if not provider_network %}
+  data_subnet:
+    type: string
+    default: ''
+    label: Subnet ID
+    description: Subnet resource
+{% endif %}
+{% endif %}
+
   secgrp:
     type: comma_delimited_list
     label: Security groups
@@ -133,6 +155,11 @@ outputs:
 {% endif %}
         - addr
 
+{% if openshift_use_flannel|default(False)|bool %}
+conditions:
+  no_data_subnet: {not: { get_param: attach_data_net} }
+{% endif %}
+
 resources:
 
   server:
@@ -143,10 +170,27 @@ resources:
       image:     { get_param: image }
       flavor:    { get_param: flavor }
       networks:
+{% if openshift_use_flannel|default(False)|bool %}
+        if:
+          - no_data_subnet
+{% if use_trunk_ports|default(false)|bool %}
+          - - port:  { get_attr: [trunk-port, port_id] }
+{% else %}
+          - - port:  { get_resource: port }
+{% endif %}
+{% if use_trunk_ports|default(false)|bool %}
+          - - port:  { get_attr: [trunk-port, port_id] }
+{% else %}
+          - - port:  { get_resource: port }
+            - port:  { get_resource: data_port }
+{% endif %}
+
+{% else %}
 {% if use_trunk_ports|default(false)|bool %}
         - port:  { get_attr: [trunk-port, port_id] }
 {% else %}
         - port:  { get_resource: port }
+{% endif %}
 {% endif %}
       user_data:
         get_file: user-data
@@ -179,6 +223,19 @@ resources:
 {% endif %}
       security_groups: { get_param: secgrp }
 
+{% if openshift_use_flannel|default(False)|bool %}
+  data_port:
+    type: OS::Neutron::Port
+    condition: { not: no_data_subnet }
+    properties:
+      network: { get_param: data_net }
+      port_security_enabled: false
+{% if not provider_network %}
+      fixed_ips:
+        - subnet: { get_param: data_subnet }
+{% endif %}
+{% endif %}
+
 {% if not provider_network %}
   floating-ip:
     type: OS::Neutron::FloatingIP
diff --git a/roles/openstack-stack/templates/heat_stack_server_nofloating.yaml.j2 b/roles/openstack-stack/templates/heat_stack_server_nofloating.yaml.j2
index 4b79d5ab6..742d53649 100644
--- a/roles/openstack-stack/templates/heat_stack_server_nofloating.yaml.j2
+++ b/roles/openstack-stack/templates/heat_stack_server_nofloating.yaml.j2
@@ -66,6 +66,26 @@ parameters:
     label: Subnet ID
     description: Subnet resource
 
+{% if openshift_use_flannel|default(False)|bool %}
+  attach_data_net:
+    type: boolean
+    default: false
+    label: Attach-data-net
+    description: A switch for data port connection
+
+  data_net:
+    type: string
+    default: ''
+    label: Net ID
+    description: Net resource
+
+  data_subnet:
+    type: string
+    default: ''
+    label: Subnet ID
+    description: Subnet resource
+{% endif %}
+
   secgrp:
     type: comma_delimited_list
     label: Security groups
@@ -110,6 +130,11 @@ outputs:
         - 0
         - addr
 
+{% if openshift_use_flannel|default(False)|bool %}
+conditions:
+  no_data_subnet: {not: { get_param: attach_data_net} }
+{% endif %}
+
 resources:
 
   server_nofloating:
@@ -120,10 +145,27 @@ resources:
       image:     { get_param: image }
       flavor:    { get_param: flavor }
       networks:
+{% if openshift_use_flannel|default(False)|bool %}
+        if:
+          - no_data_subnet
+{% if use_trunk_ports|default(false)|bool %}
+          - - port:  { get_attr: [trunk-port, port_id] }
+{% else %}
+          - - port:  { get_resource: port }
+{% endif %}
+{% if use_trunk_ports|default(false)|bool %}
+          - - port:  { get_attr: [trunk-port, port_id] }
+{% else %}
+          - - port:  { get_resource: port }
+            - port:  { get_resource: data_port }
+{% endif %}
+
+{% else %}
 {% if use_trunk_ports|default(false)|bool %}
         - port:  { get_attr: [trunk-port, port_id] }
 {% else %}
         - port:  { get_resource: port }
+{% endif %}
 {% endif %}
       user_data:
         get_file: user-data
@@ -154,6 +196,19 @@ resources:
         - subnet: { get_param: subnet }
       security_groups: { get_param: secgrp }
 
+{% if openshift_use_flannel|default(False)|bool %}
+  data_port:
+    type: OS::Neutron::Port
+    condition: { not: no_data_subnet }
+    properties:
+      network: { get_param: data_net }
+      port_security_enabled: false
+{% if not provider_network %}
+      fixed_ips:
+        - subnet: { get_param: data_subnet }
+{% endif %}
+{% endif %}
+
 {% if not ephemeral_volumes|default(false)|bool %}
   cinder_volume:
     type: OS::Cinder::Volume
-- 
cgit v1.2.3

-- 
cgit v1.2.3


From e89bd6b1cb32ad52f0109f80022e801943b51893 Mon Sep 17 00:00:00 2001
From: Bogdan Dobrelya <bdobreli@redhat.com>
Date: Tue, 17 Oct 2017 10:12:59 +0200
Subject: [WIP] Merge server with nofloating server heat templates (#761)

Merge server with nofloating server heat templates
---
 roles/openstack-stack/tasks/generate-templates.yml |   6 -
 roles/openstack-stack/templates/heat_stack.yaml.j2 |  42 ++--
 .../templates/heat_stack_server.yaml.j2            |  12 +-
 .../templates/heat_stack_server_nofloating.yaml.j2 | 225 ---------------------
 4 files changed, 35 insertions(+), 250 deletions(-)
 delete mode 100644 roles/openstack-stack/templates/heat_stack_server_nofloating.yaml.j2

diff --git a/roles/openstack-stack/tasks/generate-templates.yml b/roles/openstack-stack/tasks/generate-templates.yml
index 110da8444..0ff50a095 100644
--- a/roles/openstack-stack/tasks/generate-templates.yml
+++ b/roles/openstack-stack/tasks/generate-templates.yml
@@ -20,12 +20,6 @@
     src: heat_stack_server.yaml.j2
     dest: "{{ stack_template_pre.path }}/server.yaml"
 
-- name: generate HOT server w/o floating IPs template from jinja2 template
-  template:
-    src: heat_stack_server_nofloating.yaml.j2
-    dest: "{{ stack_template_pre.path }}/server_nofloating.yaml"
-  when: use_bastion|bool
-
 - name: generate user_data from jinja2 template
   template:
     src: user_data.j2
diff --git a/roles/openstack-stack/templates/heat_stack.yaml.j2 b/roles/openstack-stack/templates/heat_stack.yaml.j2
index 1f1e33cf2..a69b7fc00 100644
--- a/roles/openstack-stack/templates/heat_stack.yaml.j2
+++ b/roles/openstack-stack/templates/heat_stack.yaml.j2
@@ -71,6 +71,9 @@ outputs:
     value: { get_attr: [ dns, private_ip ] }
 {% endif %}
 
+conditions:
+  no_floating: {% if provider_network or use_bastion|bool %}true{% else %}false{% endif %}
+
 resources:
 
 {% if not provider_network %}
@@ -504,11 +507,7 @@ resources:
     properties:
       count: {{ num_etcd }}
       resource_def:
-{% if use_bastion|bool %}
-        type: server_nofloating.yaml
-{% else %}
         type: server.yaml
-{% endif %}
         properties:
           name:
             str_replace:
@@ -543,8 +542,13 @@ resources:
           secgrp:
             - { get_resource: {% if openstack_flat_secgrp|default(False)|bool %}flat-secgrp{% else %}etcd-secgrp{% endif %} }
             - { get_resource: common-secgrp }
-{% if not use_bastion|bool and not provider_network %}
-          floating_network: {{ external_network }}
+          floating_network:
+            if:
+              - no_floating
+              - null
+              - {{ external_network }}
+{% if use_bastion|bool or provider_network %}
+          attach_float_net: false
 {% endif %}
           volume_size: {{ etcd_volume_size }}
 {% if not provider_network %}
@@ -622,11 +626,7 @@ resources:
     properties:
       count: {{ num_masters }}
       resource_def:
-{% if use_bastion|bool %}
-        type: server_nofloating.yaml
-{% else %}
         type: server.yaml
-{% endif %}
         properties:
           name:
             str_replace:
@@ -674,8 +674,13 @@ resources:
 {% endif %}
 {% endif %}
             - { get_resource: common-secgrp }
-{% if not use_bastion|bool and not provider_network %}
-          floating_network: {{ external_network }}
+          floating_network:
+            if:
+              - no_floating
+              - null
+              - {{ external_network }}
+{% if use_bastion|bool or provider_network %}
+          attach_float_net: false
 {% endif %}
           volume_size: {{ master_volume_size }}
 {% if master_server_group_policies|length > 0 %}
@@ -694,11 +699,7 @@ resources:
       removal_policies:
       - resource_list: {{ nodes_to_remove }}
       resource_def:
-{% if use_bastion|bool %}
-        type: server_nofloating.yaml
-{% else %}
         type: server.yaml
-{% endif %}
         properties:
           name:
             str_replace:
@@ -743,8 +744,13 @@ resources:
           secgrp:
             - { get_resource: {% if openstack_flat_secgrp|default(False)|bool %}flat-secgrp{% else %}node-secgrp{% endif %} }
             - { get_resource: common-secgrp }
-{% if not use_bastion|bool and not provider_network %}
-          floating_network: {{ external_network }}
+          floating_network:
+            if:
+              - no_floating
+              - null
+              - {{ external_network }}
+{% if use_bastion|bool or provider_network %}
+          attach_float_net: false
 {% endif %}
           volume_size: {{ node_volume_size }}
 {% if not provider_network %}
diff --git a/roles/openstack-stack/templates/heat_stack_server.yaml.j2 b/roles/openstack-stack/templates/heat_stack_server.yaml.j2
index 6552e0a0d..9ffe721a5 100644
--- a/roles/openstack-stack/templates/heat_stack_server.yaml.j2
+++ b/roles/openstack-stack/templates/heat_stack_server.yaml.j2
@@ -95,9 +95,17 @@ parameters:
     label: Security groups
     description: Security group resources
 
+  attach_float_net:
+    type: boolean
+    default: true
+
+    label: Attach-float-net
+    description: A switch for floating network port connection
+
 {% if not provider_network %}
   floating_network:
     type: string
+    default: ''
     label: Floating network
     description: Network to allocate floating IP from
 {% endif %}
@@ -155,8 +163,9 @@ outputs:
 {% endif %}
         - addr
 
-{% if openshift_use_flannel|default(False)|bool %}
 conditions:
+  no_floating: {not: { get_param: attach_float_net} }
+{% if openshift_use_flannel|default(False)|bool %}
   no_data_subnet: {not: { get_param: attach_data_net} }
 {% endif %}
 
@@ -238,6 +247,7 @@ resources:
 
 {% if not provider_network %}
   floating-ip:
+    condition: { not: no_floating }
     type: OS::Neutron::FloatingIP
     properties:
       floating_network: { get_param: floating_network }
diff --git a/roles/openstack-stack/templates/heat_stack_server_nofloating.yaml.j2 b/roles/openstack-stack/templates/heat_stack_server_nofloating.yaml.j2
deleted file mode 100644
index 742d53649..000000000
--- a/roles/openstack-stack/templates/heat_stack_server_nofloating.yaml.j2
+++ /dev/null
@@ -1,225 +0,0 @@
-heat_template_version: 2016-10-14
-
-description: OpenShift cluster server w/o floating IP
-
-parameters:
-
-  name:
-    type: string
-    label: Name
-    description: Name
-
-  group:
-    type: string
-    label: Host Group
-    description: The Primary Ansible Host Group
-    default: host
-
-  cluster_env:
-    type: string
-    label: Cluster environment
-    description: Environment of the cluster
-
-  cluster_id:
-    type: string
-    label: Cluster ID
-    description: Identifier of the cluster
-
-  type:
-    type: string
-    label: Type
-    description: Type master or node
-
-  subtype:
-    type: string
-    label: Sub-type
-    description: Sub-type compute or infra for nodes, default otherwise
-    default: default
-
-  key_name:
-    type: string
-    label: Key name
-    description: Key name of keypair
-
-  image:
-    type: string
-    label: Image
-    description: Name of the image
-
-  flavor:
-    type: string
-    label: Flavor
-    description: Name of the flavor
-
-  net:
-    type: string
-    label: Net ID
-    description: Net resource
-
-  net_name:
-    type: string
-    label: Net name
-    description: Net name
-
-  subnet:
-    type: string
-    label: Subnet ID
-    description: Subnet resource
-
-{% if openshift_use_flannel|default(False)|bool %}
-  attach_data_net:
-    type: boolean
-    default: false
-    label: Attach-data-net
-    description: A switch for data port connection
-
-  data_net:
-    type: string
-    default: ''
-    label: Net ID
-    description: Net resource
-
-  data_subnet:
-    type: string
-    default: ''
-    label: Subnet ID
-    description: Subnet resource
-{% endif %}
-
-  secgrp:
-    type: comma_delimited_list
-    label: Security groups
-    description: Security group resources
-
-  availability_zone:
-    type: string
-    description: The Availability Zone to launch the instance.
-    default: nova
-
-  volume_size:
-    type: number
-    description: Size of the volume to be created.
-    default: 1
-    constraints:
-      - range: { min: 1, max: 1024 }
-        description: must be between 1 and 1024 Gb.
-
-  node_labels:
-    type: json
-    description: OpenShift Node Labels
-    default: {"region": "default" }
-
-  scheduler_hints:
-    type: json
-    description: Server scheduler hints.
-    default: {}
-
-outputs:
-
-  name:
-    description: Name of the server
-    value: { get_attr: [ server_nofloating, name ] }
-
-  private_ip:
-    description: Private IP of the server
-    value:
-      get_attr:
-        - server_nofloating
-        - addresses
-        - { get_param: net_name }
-        - 0
-        - addr
-
-{% if openshift_use_flannel|default(False)|bool %}
-conditions:
-  no_data_subnet: {not: { get_param: attach_data_net} }
-{% endif %}
-
-resources:
-
-  server_nofloating:
-    type: OS::Nova::Server
-    properties:
-      name:      { get_param: name }
-      key_name:  { get_param: key_name }
-      image:     { get_param: image }
-      flavor:    { get_param: flavor }
-      networks:
-{% if openshift_use_flannel|default(False)|bool %}
-        if:
-          - no_data_subnet
-{% if use_trunk_ports|default(false)|bool %}
-          - - port:  { get_attr: [trunk-port, port_id] }
-{% else %}
-          - - port:  { get_resource: port }
-{% endif %}
-{% if use_trunk_ports|default(false)|bool %}
-          - - port:  { get_attr: [trunk-port, port_id] }
-{% else %}
-          - - port:  { get_resource: port }
-            - port:  { get_resource: data_port }
-{% endif %}
-
-{% else %}
-{% if use_trunk_ports|default(false)|bool %}
-        - port:  { get_attr: [trunk-port, port_id] }
-{% else %}
-        - port:  { get_resource: port }
-{% endif %}
-{% endif %}
-      user_data:
-        get_file: user-data
-      user_data_format: RAW
-      user_data_update_policy: IGNORE
-      metadata:
-        group: { get_param: group }
-        environment: { get_param: cluster_env }
-        clusterid: { get_param: cluster_id }
-        host-type: { get_param: type }
-        sub-host-type:    { get_param: subtype }
-        node_labels: { get_param: node_labels }
-      scheduler_hints: { get_param: scheduler_hints }
-
-{% if use_trunk_ports|default(false)|bool %}
-  trunk-port:
-    type: OS::Neutron::Trunk
-    properties:
-      name: { get_param: name }
-      port: { get_resource: port }
-{% endif %}
-
-  port:
-    type: OS::Neutron::Port
-    properties:
-      network: { get_param: net }
-      fixed_ips:
-        - subnet: { get_param: subnet }
-      security_groups: { get_param: secgrp }
-
-{% if openshift_use_flannel|default(False)|bool %}
-  data_port:
-    type: OS::Neutron::Port
-    condition: { not: no_data_subnet }
-    properties:
-      network: { get_param: data_net }
-      port_security_enabled: false
-{% if not provider_network %}
-      fixed_ips:
-        - subnet: { get_param: data_subnet }
-{% endif %}
-{% endif %}
-
-{% if not ephemeral_volumes|default(false)|bool %}
-  cinder_volume:
-    type: OS::Cinder::Volume
-    properties:
-      size: { get_param: volume_size }
-      availability_zone: { get_param: availability_zone }
-
-  volume_attachment:
-    type: OS::Cinder::VolumeAttachment
-    properties:
-      volume_id: { get_resource: cinder_volume }
-      instance_uuid: { get_resource: server_nofloating }
-      mountpoint: /dev/sdb
-{% endif %}
-- 
cgit v1.2.3

-- 
cgit v1.2.3


From 8d14089a84119d4b824bfea991099941122a2c12 Mon Sep 17 00:00:00 2001
From: Chandler Wilkerson <cwilkers@redhat.com>
Date: Tue, 17 Oct 2017 07:53:18 -0500
Subject: Docker storage fix (#812)

* Added task to stop docker before templating config

* Rearranged storage roles in rhv install
---
 roles/docker-storage-setup/tasks/main.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/roles/docker-storage-setup/tasks/main.yaml b/roles/docker-storage-setup/tasks/main.yaml
index 209062ca7..dbff85662 100644
--- a/roles/docker-storage-setup/tasks/main.yaml
+++ b/roles/docker-storage-setup/tasks/main.yaml
@@ -1,4 +1,7 @@
 ---
+- name: stop docker
+  service: name=docker state=stopped
+
 - block:
     - name: create the docker-storage config file
       template:
@@ -7,7 +10,6 @@
         owner: root
         group: root
         mode: 0644
-
   when:
     - ansible_distribution_version | version_compare('7.4', '>=')
     - ansible_distribution == "RedHat"
@@ -20,7 +22,6 @@
         owner: root
         group: root
         mode: 0644
-
   when:
     - ansible_distribution_version | version_compare('7.4', '<')
     - ansible_distribution == "RedHat"
-- 
cgit v1.2.3

-- 
cgit v1.2.3


From d2ff422b284f04b8a19ad4c6aa388ba397d915e1 Mon Sep 17 00:00:00 2001
From: Bogdan Dobrelya <bdobreli@redhat.com>
Date: Wed, 18 Oct 2017 12:53:31 +0200
Subject: Add Flannel support (#814)

* Add flannel support

* Document Flannel SDN use case for a separate data network.
* Add post install step for flannel SDN
* Configure iptables rules as described for OCP 3.4 refarch
  https://access.redhat.com/documentation/en-us/reference_architectures/2017/html/deploying_red_hat_openshift_container_platform_3.4_on_red_hat_openstack_platform_10/emphasis_manual_deployment_emphasis#run_ansible_installer

* Configure flannel interface options

Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>

* Use os_firewall from galaxy for required flannel rules

For flannel SDN:
* Add openshift-ansible as a galaxy dependency module.
* Use openshift-ansible/roles/os_firewall to apply DNS rules
  for flanel SDN.
* Apply the remaining advanced rules with direct
  iptables commands as os_firewall do not support advanced rules.
* Persist only iptables rules w/o dynamic KUBe rules. Those are
  added runtime and need restoration after reboot or iptables restart.
* Configure and enable the masked iptables service on the app nodes.
  Enable it to allow the in-memory rules to be persisted.
  Disable firewalld, which is the expected default behavior of the
  os_firewall module.

Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>

* Allow access from nodes to masters' port 2379 when using flannel

Flannel requires to gather information from etcd to configure and
assign the subnets in the nodes, therefore, allow access from nodes to port 2379/tcp to the master security group.

Signed-off-by: Bogdan Dobrelya <bdobreli@redhat.com>
---
 playbooks/provisioning/openstack/README.md         | 20 ++++++++
 .../openstack/galaxy-requirements.yaml             |  4 ++
 playbooks/provisioning/openstack/post-install.yml  | 57 ++++++++++++++++++++++
 .../openstack/post-provision-openstack.yml         | 25 ++++++++++
 .../sample-inventory/group_vars/OSEv3.yml          |  2 +
 roles/openstack-stack/templates/heat_stack.yaml.j2 |  6 +++
 6 files changed, 114 insertions(+)
 create mode 100644 playbooks/provisioning/openstack/post-install.yml

diff --git a/playbooks/provisioning/openstack/README.md b/playbooks/provisioning/openstack/README.md
index b9a3b23de..a277047e1 100644
--- a/playbooks/provisioning/openstack/README.md
+++ b/playbooks/provisioning/openstack/README.md
@@ -145,6 +145,26 @@ via the public IP of the server. You can not send updates via the private
 IP yet. This forces the in-stack private server to have a floating IP.
 See also the [security notes](#security-notes)
 
+#### Flannel networking
+
+In order to configure the
+[flannel networking](https://docs.openshift.com/container-platform/3.6/install_config/configuring_sdn.html#using-flannel),
+uncomment and adjust the appropriate `inventory/group_vars/OSEv3.yml` group vars.
+Note that the `osm_cluster_network_cidr` must not overlap with the default
+Docker bridge subnet of 172.17.0.0/16. Or you should change the docker0 default
+CIDR range otherwise. For example, by adding `--bip=192.168.2.1/24` to
+`DOCKER_NETWORK_OPTIONS` located in `/etc/sysconfig/docker-network`.
+
+Also note that the flannel network will be provisioned on a separate isolated Neutron
+subnet defined from `osm_cluster_network_cidr` and having ports security disabled.
+Use the `openstack_private_data_network_name` variable to define the network
+name for the heat stack resource.
+
+After the cluster deployment done, you should run an additional post installation
+step for flannel and docker iptables configuration:
+
+   ansible-playbook openshift-ansible-contrib/playbooks/provisioning/openstack/post-install.yml
+
 #### Other configuration variables
 
 `openstack_ssh_key` is a Nova keypair - you can see your keypairs with
diff --git a/playbooks/provisioning/openstack/galaxy-requirements.yaml b/playbooks/provisioning/openstack/galaxy-requirements.yaml
index 93dd14ec2..1d745dcc3 100644
--- a/playbooks/provisioning/openstack/galaxy-requirements.yaml
+++ b/playbooks/provisioning/openstack/galaxy-requirements.yaml
@@ -4,3 +4,7 @@
 # From 'infra-ansible'
 - src: https://github.com/redhat-cop/infra-ansible
   version: master
+
+# From 'openshift-ansible'
+- src: https://github.com/openshift/openshift-ansible
+  version: master
diff --git a/playbooks/provisioning/openstack/post-install.yml b/playbooks/provisioning/openstack/post-install.yml
new file mode 100644
index 000000000..417813e2a
--- /dev/null
+++ b/playbooks/provisioning/openstack/post-install.yml
@@ -0,0 +1,57 @@
+---
+- hosts: OSEv3
+  gather_facts: False
+  become: True
+  tasks:
+    - name: Save iptables rules to a backup file
+      when: openshift_use_flannel|default(False)|bool
+      shell: iptables-save > /etc/sysconfig/iptables.orig-$(date +%Y%m%d%H%M%S)
+
+# Enable iptables service on app nodes to persist custom rules (flannel SDN)
+# FIXME(bogdando) w/a https://bugzilla.redhat.com/show_bug.cgi?id=1490820
+- hosts: app
+  gather_facts: False
+  become: True
+  vars:
+    os_firewall_allow:
+      - service: dnsmasq tcp
+        port: 53/tcp
+      - service: dnsmasq udp
+        port: 53/udp
+  tasks:
+    - when: openshift_use_flannel|default(False)|bool
+      block:
+        - include_role:
+            name: openshift-ansible/roles/os_firewall
+        - include_role:
+            name: openshift-ansible/roles/lib_os_firewall
+        - name: set allow rules for dnsmasq
+          os_firewall_manage_iptables:
+            name: "{{ item.service }}"
+            action: add
+            protocol: "{{ item.port.split('/')[1] }}"
+            port: "{{ item.port.split('/')[0] }}"
+          with_items: "{{ os_firewall_allow }}"
+
+- hosts: OSEv3
+  gather_facts: False
+  become: True
+  tasks:
+    - name: Apply post-install iptables hacks for Flannel SDN (the best effort)
+      when: openshift_use_flannel|default(False)|bool
+      block:
+        - name: set allow/masquerade rules for for flannel/docker
+          shell: >-
+            (iptables-save | grep -q custom-flannel-docker-1) ||
+            iptables -A DOCKER -w
+            -p all -j ACCEPT
+            -m comment --comment "custom-flannel-docker-1";
+            (iptables-save | grep -q custom-flannel-docker-2) ||
+            iptables -t nat -A POSTROUTING -w
+            -o {{flannel_interface|default('eth1')}}
+            -m comment --comment "custom-flannel-docker-2"
+            -j MASQUERADE
+
+        # NOTE(bogdando) the rules will not be restored, when iptables service unit is disabled & masked
+        - name: Persist in-memory iptables rules (w/o dynamic KUBE rules)
+          shell: iptables-save | grep -v KUBE > /etc/sysconfig/iptables
diff --git a/playbooks/provisioning/openstack/post-provision-openstack.yml b/playbooks/provisioning/openstack/post-provision-openstack.yml
index a80e8d829..e460fbf12 100644
--- a/playbooks/provisioning/openstack/post-provision-openstack.yml
+++ b/playbooks/provisioning/openstack/post-provision-openstack.yml
@@ -76,6 +76,16 @@
   hosts: OSEv3
   gather_facts: true
   become: true
+  vars:
+    interface: "{{ flannel_interface|default('eth1') }}"
+    interface_file: /etc/sysconfig/network-scripts/ifcfg-{{ interface }}
+    interface_config:
+      DEVICE: "{{ interface }}"
+      TYPE: Ethernet
+      BOOTPROTO: dhcp
+      ONBOOT: 'yes'
+      DEFTROUTE: 'no'
+      PEERDNS: 'no'
   pre_tasks:
     - name: "Include DNS configuration to ensure proper name resolution"
       lineinfile:
@@ -83,6 +93,21 @@
         dest: /etc/sysconfig/network
         regexp: "IP4_NAMESERVERS={{ hostvars['localhost'].private_dns_server }}"
         line: "IP4_NAMESERVERS={{ hostvars['localhost'].private_dns_server }}"
+    - name: "Configure the flannel interface options"
+      when: openshift_use_flannel|default(False)|bool
+      block:
+        - file:
+            dest: "{{ interface_file }}"
+            state: touch
+            mode: 0644
+            owner: root
+            group: root
+        - lineinfile:
+            state: present
+            dest: "{{ interface_file }}"
+            regexp: "{{ item.key }}="
+            line: "{{ item.key }}={{ item.value }}"
+          with_dict: "{{ interface_config }}"
   roles:
     - node-network-manager
 
diff --git a/playbooks/provisioning/openstack/sample-inventory/group_vars/OSEv3.yml b/playbooks/provisioning/openstack/sample-inventory/group_vars/OSEv3.yml
index 70e77662d..949a323a7 100644
--- a/playbooks/provisioning/openstack/sample-inventory/group_vars/OSEv3.yml
+++ b/playbooks/provisioning/openstack/sample-inventory/group_vars/OSEv3.yml
@@ -53,5 +53,7 @@ openshift_override_hostname_check: true
 ansible_become: true
 
 # # Flannel networking
+#osm_cluster_network_cidr: 10.128.0.0/14
 #openshift_use_openshift_sdn: false
 #openshift_use_flannel: true
+#flannel_interface: eth1
diff --git a/roles/openstack-stack/templates/heat_stack.yaml.j2 b/roles/openstack-stack/templates/heat_stack.yaml.j2
index a69b7fc00..2359842a5 100644
--- a/roles/openstack-stack/templates/heat_stack.yaml.j2
+++ b/roles/openstack-stack/templates/heat_stack.yaml.j2
@@ -341,6 +341,12 @@ resources:
           protocol: tcp
           port_range_min: 9090
           port_range_max: 9090
+{% if openshift_use_flannel|default(False)|bool %}
+        - direction: ingress
+          protocol: tcp
+          port_range_min: 2379
+          port_range_max: 2379
+{% endif %}
 
   etcd-secgrp:
     type: OS::Neutron::SecurityGroup
-- 
cgit v1.2.3


From 3823c72af11f77b9639176921b398fbab2ac04fd Mon Sep 17 00:00:00 2001
From: Tlacenka <tlacencin@gmail.com>
Date: Wed, 18 Oct 2017 12:55:58 +0200
Subject: Add Extra CAs (custom post-provision action) (#801)

* add cas: playbook adding new CAs created

* add CAs: README updated, bug fixes

* README: improvements

* README: minor fixes

* README: removed code snippet

* README: fix
---
 playbooks/provisioning/openstack/README.md                  | 11 +++++++++++
 playbooks/provisioning/openstack/custom-actions/add-cas.yml | 13 +++++++++++++
 2 files changed, 24 insertions(+)
 create mode 100644 playbooks/provisioning/openstack/custom-actions/add-cas.yml

diff --git a/playbooks/provisioning/openstack/README.md b/playbooks/provisioning/openstack/README.md
index a277047e1..f11a9bd73 100644
--- a/playbooks/provisioning/openstack/README.md
+++ b/playbooks/provisioning/openstack/README.md
@@ -629,6 +629,16 @@ Example usage:
 ansible-playbook -i <inventory> openshift-ansible-contrib/playbooks/provisioning/openstack/custom-actions/add-docker-registry.yml  --extra-vars '{"registries": "reg1", "insecure_registries": ["ins_reg1","ins_reg2"]}'
 ```
 
+#### Adding extra CAs to the trust chain
+
+This playbook is also located in the [custom-actions](https://github.com/openshift/openshift-ansible-contrib/blob/master/playbooks/provisioning/openstack/custom-actions) directory.
+It copies passed CAs to the trust chain location and updates the trust chain on each selected host.
+
+Example usage:
+```
+ansible-playbook -i <inventory> openshift-ansible-contrib/playbooks/provisioning/openstack/custom-actions/add-cas.yml --extra-vars '{"ca_files": [<absolute path to ca1 file>, <absolute path to ca2 file>]}'
+```
+
 Please consider contributing your custom playbook back to openshift-ansible-contrib!
 
 A library of custom post-provision actions exists in `openshift-ansible-contrib/playbooks/provisioning/openstack/custom-actions`. Playbooks include:
@@ -636,6 +646,7 @@ A library of custom post-provision actions exists in `openshift-ansible-contrib/
 * [add-yum-repos.yml](https://github.com/openshift/openshift-ansible-contrib/blob/master/playbooks/provisioning/openstack/custom-actions/add-yum-repos.yml): adds a list of custom yum repositories to every node in the cluster
 * [add-rhn-pools.yml](https://github.com/openshift/openshift-ansible-contrib/blob/master/playbooks/provisioning/openstack/custom-actions/add-rhn-pools.yml): attaches a list of additional RHN pools to every node in the cluster
 * [add-docker-registry.yml](https://github.com/openshift/openshift-ansible-contrib/blob/master/playbooks/provisioning/openstack/custom-actions/add-docker-registry.yml): adds a list of docker registries to the docker configuration on every node in the cluster
+* [add-cas.yml](https://github.com/openshift/openshift-ansible-contrib/blob/master/playbooks/provisioning/openstack/custom-actions/add-rhn-pools.yml): adds a list of CAs to the trust chain on every node in the cluster
 
 ### Install OpenShift
 
diff --git a/playbooks/provisioning/openstack/custom-actions/add-cas.yml b/playbooks/provisioning/openstack/custom-actions/add-cas.yml
new file mode 100644
index 000000000..b2c195f91
--- /dev/null
+++ b/playbooks/provisioning/openstack/custom-actions/add-cas.yml
@@ -0,0 +1,13 @@
+---
+- hosts: cluster_hosts
+  become: true
+  vars:
+    ca_files: []
+  tasks:
+  - name: Copy CAs to the trusted CAs location
+    with_items: "{{ ca_files }}"
+    copy:
+      src: "{{ item }}"
+      dest: /etc/pki/ca-trust/source/anchors/
+  - name: Update trusted CAs
+    shell: 'update-ca-trust enable && update-ca-trust extract'
-- 
cgit v1.2.3