Back to Repositories

Validating Git-Sync Worker Configuration in Apache Airflow

This test suite validates the Git-Sync worker functionality in Apache Airflow’s Helm chart deployment. It ensures proper configuration and integration of Git synchronization with Airflow workers, covering volume mounting, container setup, and SSH authentication scenarios.

Test Coverage Overview

The test suite provides comprehensive coverage of Git-Sync worker functionality in Airflow Helm deployments.

Key areas tested include:

Volume mounting configurations for different persistence scenarios
Git-Sync container addition and management
Environment variable handling
SSH key authentication setup
Resource allocation and management

Implementation Analysis

The testing approach uses a systematic verification of Helm chart rendering outcomes. It leverages the render_chart utility and jmespath queries to validate the generated Kubernetes manifests.

Test patterns focus on:

Configuration permutations between Git-Sync and persistence settings
Container specification validation
Resource configuration verification
Security credential handling

Technical Details

Testing tools and components include:

jmespath for JSON/YAML parsing and validation
Helm template generator for manifest rendering
Python unit test framework
Mock configurations for various deployment scenarios

Best Practices Demonstrated

The test suite exemplifies several testing best practices:

Comprehensive scenario coverage including edge cases
Systematic validation of configuration parameters
Security-focused testing for credential handling
Clear test case organization and naming
Modular test structure for maintainability

apache/airflow

helm_tests/other/test_git_sync_worker.py

            
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
from __future__ import annotations

import jmespath

from tests.charts.helm_template_generator import render_chart


class TestGitSyncWorker:
    """Test git sync worker."""

    def test_should_add_dags_volume_to_the_worker_if_git_sync_and_persistence_is_enabled(self):
        docs = render_chart(
            values={
                "executor": "CeleryExecutor",
                "dags": {"persistence": {"enabled": True}, "gitSync": {"enabled": True}},
            },
            show_only=["templates/workers/worker-deployment.yaml"],
        )

        assert jmespath.search("spec.template.spec.volumes[0].name", docs[0]) == "config"
        assert jmespath.search("spec.template.spec.volumes[1].name", docs[0]) == "dags"

    def test_should_add_dags_volume_to_the_worker_if_git_sync_is_enabled_and_persistence_is_disabled(self):
        docs = render_chart(
            values={
                "executor": "CeleryExecutor",
                "dags": {"gitSync": {"enabled": True}, "persistence": {"enabled": False}},
            },
            show_only=["templates/workers/worker-deployment.yaml"],
        )

        assert jmespath.search("spec.template.spec.volumes[0].name", docs[0]) == "config"
        assert jmespath.search("spec.template.spec.volumes[1].name", docs[0]) == "dags"

    def test_should_add_git_sync_container_to_worker_if_persistence_is_not_enabled_but_git_sync_is(self):
        docs = render_chart(
            values={
                "executor": "CeleryExecutor",
                "dags": {
                    "gitSync": {"enabled": True, "containerName": "git-sync"},
                    "persistence": {"enabled": False},
                },
            },
            show_only=["templates/workers/worker-deployment.yaml"],
        )

        assert jmespath.search("spec.template.spec.containers[1].name", docs[0]) == "git-sync"

    def test_should_not_add_sync_container_to_worker_if_git_sync_and_persistence_are_enabled(self):
        docs = render_chart(
            values={
                "executor": "CeleryExecutor",
                "dags": {
                    "gitSync": {"enabled": True, "containerName": "git-sync"},
                    "persistence": {"enabled": True},
                },
            },
            show_only=["templates/workers/worker-deployment.yaml"],
        )

        assert jmespath.search("spec.template.spec.containers[1].name", docs[0]) != "git-sync"

    def test_should_add_env(self):
        docs = render_chart(
            values={
                "dags": {
                    "gitSync": {
                        "enabled": True,
                        "env": [{"name": "FOO", "value": "bar"}],
                    }
                },
            },
            show_only=["templates/workers/worker-deployment.yaml"],
        )

        assert {"name": "FOO", "value": "bar"} in jmespath.search(
            "spec.template.spec.containers[1].env", docs[0]
        )

    def test_resources_are_configurable(self):
        docs = render_chart(
            values={
                "dags": {
                    "gitSync": {
                        "enabled": True,
                        "resources": {
                            "limits": {"cpu": "200m", "memory": "128Mi"},
                            "requests": {"cpu": "300m", "memory": "169Mi"},
                        },
                    },
                },
            },
            show_only=["templates/workers/worker-deployment.yaml"],
        )
        assert jmespath.search("spec.template.spec.containers[1].resources.limits.memory", docs[0]) == "128Mi"
        assert (
            jmespath.search("spec.template.spec.containers[1].resources.requests.memory", docs[0]) == "169Mi"
        )
        assert jmespath.search("spec.template.spec.containers[1].resources.requests.cpu", docs[0]) == "300m"

    def test_validate_sshkeysecret_not_added_when_persistence_is_enabled(self):
        docs = render_chart(
            values={
                "dags": {
                    "gitSync": {
                        "enabled": True,
                        "containerName": "git-sync-test",
                        "sshKeySecret": "ssh-secret",
                        "knownHosts": None,
                        "branch": "test-branch",
                    },
                    "persistence": {"enabled": True},
                }
            },
            show_only=["templates/workers/worker-deployment.yaml"],
        )

        assert "git-sync-ssh-key" not in jmespath.search("spec.template.spec.volumes[].name", docs[0])

    def test_validate_if_ssh_params_are_added_with_git_ssh_key(self):
        docs = render_chart(
            values={
                "dags": {
                    "gitSync": {
                        "enabled": True,
                        "sshKey": "dummy-ssh-key",
                    }
                }
            },
            show_only=["templates/workers/worker-deployment.yaml"],
        )

        assert {"name": "GIT_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search(
            "spec.template.spec.containers[1].env", docs[0]
        )
        assert {"name": "GITSYNC_SSH_KEY_FILE", "value": "/etc/git-secret/ssh"} in jmespath.search(
            "spec.template.spec.containers[1].env", docs[0]
        )
        assert {"name": "GIT_SYNC_SSH", "value": "true"} in jmespath.search(
            "spec.template.spec.containers[1].env", docs[0]
        )
        assert {"name": "GITSYNC_SSH", "value": "true"} in jmespath.search(
            "spec.template.spec.containers[1].env", docs[0]
        )
        assert {"name": "GIT_KNOWN_HOSTS", "value": "false"} in jmespath.search(
            "spec.template.spec.containers[1].env", docs[0]
        )
        assert {"name": "GITSYNC_SSH_KNOWN_HOSTS", "value": "false"} in jmespath.search(
            "spec.template.spec.containers[1].env", docs[0]
        )
        assert {
            "name": "git-sync-ssh-key",
            "secret": {"secretName": "release-name-ssh-secret", "defaultMode": 288},
        } in jmespath.search("spec.template.spec.volumes", docs[0])

    def test_container_lifecycle_hooks(self):
        docs = render_chart(
            values={
                "dags": {
                    "gitSync": {
                        "enabled": True,
                        "containerLifecycleHooks": {
                            "postStart": {
                                "exec": {
                                    "command": [
                                        "/bin/sh",
                                        "-c",
                                        "echo postStart handler > /git/message_start",
                                    ]
                                }
                            },
                            "preStop": {
                                "exec": {
                                    "command": ["/bin/sh", "-c", "echo preStop handler > /git/message_start"]
                                }
                            },
                        },
                    },
                },
            },
            show_only=["templates/workers/worker-deployment.yaml"],
        )
        assert jmespath.search("spec.template.spec.containers[1].lifecycle", docs[0]) == {
            "postStart": {
                "exec": {"command": ["/bin/sh", "-c", "echo postStart handler > /git/message_start"]}
            },
            "preStop": {"exec": {"command": ["/bin/sh", "-c", "echo preStop handler > /git/message_start"]}},
        }