Skip to content

Image Builders

zenml.image_builders special

Image builders allow you to build container images.

base_image_builder

Base class for all ZenML image builders.

BaseImageBuilder (StackComponent, ABC)

Base class for all ZenML image builders.

Source code in zenml/image_builders/base_image_builder.py 
class BaseImageBuilder(StackComponent, ABC):
    """Base class for all ZenML image builders."""

    @property
    def config(self) -> BaseImageBuilderConfig:
        """The stack component configuration.

        Returns:
            The configuration.
        """
        return cast(BaseImageBuilderConfig, self._config)

    @property
    def build_context_class(self) -> Type["BuildContext"]:
        """Build context class to use.

        The default build context class creates a build context that works
        for the Docker daemon. Override this method if your image builder
        requires a custom context.

        Returns:
            The build context class.
        """
        from zenml.image_builders import BuildContext

        return BuildContext

    @abstractmethod
    def build(
        self,
        image_name: str,
        build_context: "BuildContext",
        docker_build_options: Dict[str, Any],
        container_registry: Optional["BaseContainerRegistry"] = None,
    ) -> str:
        """Builds a Docker image.

        If a container registry is passed, the image will be pushed to that
        registry.

        Args:
            image_name: Name of the image to build.
            build_context: The build context to use for the image.
            docker_build_options: Docker build options.
            container_registry: Optional container registry to push to.

        Returns:
            The Docker image repo digest or name.
        """
build_context_class: Type[BuildContext] property readonly

Build context class to use.

The default build context class creates a build context that works for the Docker daemon. Override this method if your image builder requires a custom context.

Returns:

Type Description
Type[BuildContext]

The build context class.
config: BaseImageBuilderConfig property readonly

The stack component configuration.

Returns:

Type Description
BaseImageBuilderConfig

The configuration.
build(self, image_name, build_context, docker_build_options, container_registry=None)

Builds a Docker image.

If a container registry is passed, the image will be pushed to that registry.

Parameters:

Name Type Description Default
image_name str

Name of the image to build.

 required
build_context BuildContext

The build context to use for the image.

 required
docker_build_options Dict[str, Any]

Docker build options.

 required
container_registry Optional[BaseContainerRegistry]

Optional container registry to push to.

 None

Returns:

Type Description
str

The Docker image repo digest or name.
Source code in zenml/image_builders/base_image_builder.py 
@abstractmethod
def build(
    self,
    image_name: str,
    build_context: "BuildContext",
    docker_build_options: Dict[str, Any],
    container_registry: Optional["BaseContainerRegistry"] = None,
) -> str:
    """Builds a Docker image.

    If a container registry is passed, the image will be pushed to that
    registry.

    Args:
        image_name: Name of the image to build.
        build_context: The build context to use for the image.
        docker_build_options: Docker build options.
        container_registry: Optional container registry to push to.

    Returns:
        The Docker image repo digest or name.
    """

BaseImageBuilderConfig (StackComponentConfig) pydantic-model

Base config for image builders.

Source code in zenml/image_builders/base_image_builder.py 
class BaseImageBuilderConfig(StackComponentConfig):
    """Base config for image builders."""

BaseImageBuilderFlavor (Flavor, ABC)

Base class for all ZenML image builder flavors.

Source code in zenml/image_builders/base_image_builder.py 
class BaseImageBuilderFlavor(Flavor, ABC):
    """Base class for all ZenML image builder flavors."""

    @property
    def type(self) -> StackComponentType:
        """Returns the flavor type.

        Returns:
            The flavor type.
        """
        return StackComponentType.IMAGE_BUILDER

    @property
    def config_class(self) -> Type[BaseImageBuilderConfig]:
        """Config class.

        Returns:
            The config class.
        """
        return BaseImageBuilderConfig

    @property
    def implementation_class(self) -> Type[BaseImageBuilder]:
        """Implementation class.

        Returns:
            The implementation class.
        """
        return BaseImageBuilder
config_class: Type[zenml.image_builders.base_image_builder.BaseImageBuilderConfig] property readonly

Config class.

Returns:

Type Description
Type[zenml.image_builders.base_image_builder.BaseImageBuilderConfig]

The config class.
implementation_class: Type[zenml.image_builders.base_image_builder.BaseImageBuilder] property readonly

Implementation class.

Returns:

Type Description
Type[zenml.image_builders.base_image_builder.BaseImageBuilder]

The implementation class.
type: StackComponentType property readonly

Returns the flavor type.

Returns:

Type Description
StackComponentType

The flavor type.

build_context

Image build context.

BuildContext

Image build context.

This class is responsible for creating an archive of the files needed to build a container image.

Source code in zenml/image_builders/build_context.py 
class BuildContext:
    """Image build context.

    This class is responsible for creating an archive of the files needed to
    build a container image.
    """

    def __init__(
        self,
        root: Optional[str] = None,
        dockerignore_file: Optional[str] = None,
    ) -> None:
        """Initializes a build context.

        Args:
            root: Optional root directory for the build context.
            dockerignore_file: Optional path to a dockerignore file. If not
                given, a file called `.dockerignore` in the build context root
                directory will be used instead if it exists.
        """
        self._root = root
        self._dockerignore_file = dockerignore_file
        self._extra_files: Dict[str, str] = {}

    @property
    def dockerignore_file(self) -> Optional[str]:
        """The dockerignore file to use.

        Returns:
            Path to the dockerignore file to use.
        """
        if self._dockerignore_file:
            return self._dockerignore_file

        if self._root:
            default_dockerignore_path = os.path.join(
                self._root, ".dockerignore"
            )
            if fileio.exists(default_dockerignore_path):
                return default_dockerignore_path

        return None

    def add_file(self, source: str, destination: str) -> None:
        """Adds a file to the build context.

        Args:
            source: The source of the file to add. This can either be a path
                or the file content.
            destination: The path inside the build context where the file
                should be added.
        """
        if fileio.exists(source):
            with fileio.open(source) as f:
                self._extra_files[destination] = f.read()
        else:
            self._extra_files[destination] = source

    def add_directory(self, source: str, destination: str) -> None:
        """Adds a directory to the build context.

        Args:
            source: Path to the directory.
            destination: The path inside the build context where the directory
                should be added.

        Raises:
            ValueError: If `source` does not point to a directory.
        """
        if not fileio.isdir(source):
            raise ValueError(
                f"Can't add directory {source} to the build context as it "
                "does not exist or is not a directory."
            )

        for dir, _, files in fileio.walk(source):
            dir_path = Path(fileio.convert_to_str(dir))
            for file_name in files:
                file_name = fileio.convert_to_str(file_name)
                file_source = dir_path / file_name
                file_destination = (
                    Path(destination)
                    / dir_path.relative_to(source)
                    / file_name
                )

                with file_source.open("r") as f:
                    self._extra_files[file_destination.as_posix()] = f.read()

    def write_archive(self, output_file: IO[bytes], gzip: bool = True) -> None:
        """Writes an archive of the build context to the given file.

        Args:
            output_file: The file to write the archive to.
            gzip: Whether to use `gzip` to compress the file.
        """
        from docker.utils import build as docker_build_utils

        files = self._get_files()
        extra_files = self._get_extra_files()

        context_archive = docker_build_utils.create_archive(
            fileobj=output_file,
            root=self._root,
            files=sorted(files),
            gzip=gzip,
            extra_files=extra_files,
        )

        build_context_size = os.path.getsize(context_archive.name)
        if (
            self._root
            and build_context_size > 50 * 1024 * 1024
            and not self.dockerignore_file
        ):
            # The build context exceeds 50MiB and we didn't find any excludes
            # in dockerignore files -> remind to specify a .dockerignore file
            logger.warning(
                "Build context size for docker image: `%s`. If you believe this is "
                "unreasonably large, make sure to include a `.dockerignore` file "
                "at the root of your build context `%s` or specify a custom file "
                "in the Docker configuration when defining your pipeline.",
                string_utils.get_human_readable_filesize(build_context_size),
                os.path.join(self._root, ".dockerignore"),
            )

    def _get_files(self) -> Set[str]:
        """Gets all non-ignored files in the build context root directory.

        Returns:
            All build context files.
        """
        if self._root:
            exclude_patterns = self._get_exclude_patterns()
            from docker.utils import build as docker_build_utils

            return cast(
                Set[str],
                docker_build_utils.exclude_paths(
                    self._root, patterns=exclude_patterns
                ),
            )
        else:
            return set()

    def _get_extra_files(self) -> List[Tuple[str, str]]:
        """Gets all extra files of the build context.

        Returns:
            A tuple (path, file_content) for all extra files in the build
            context.
        """
        return list(self._extra_files.items())

    def _get_exclude_patterns(self) -> List[str]:
        """Gets all exclude patterns from the dockerignore file.

        Returns:
            The exclude patterns from the dockerignore file.
        """
        dockerignore = self.dockerignore_file
        if dockerignore:
            return self._parse_dockerignore(dockerignore)
        else:
            logger.info(
                "No `.dockerignore` found, including all files inside build "
                "context.",
            )
            return []

    @staticmethod
    def _parse_dockerignore(dockerignore_path: str) -> List[str]:
        """Parses a dockerignore file and returns a list of patterns to ignore.

        Args:
            dockerignore_path: Path to the dockerignore file.

        Returns:
            List of patterns to ignore.
        """
        try:
            file_content = io_utils.read_file_contents_as_string(
                dockerignore_path
            )
        except FileNotFoundError:
            logger.warning(
                "Unable to find dockerignore file at path '%s'.",
                dockerignore_path,
            )
            return []

        exclude_patterns = []
        for line in file_content.split("\n"):
            line = line.strip()
            if line and not line.startswith("#"):
                exclude_patterns.append(line)

        return exclude_patterns
dockerignore_file: Optional[str] property readonly

The dockerignore file to use.

Returns:

Type Description
Optional[str]

Path to the dockerignore file to use.
__init__(self, root=None, dockerignore_file=None) special

Initializes a build context.

Parameters:

Name Type Description Default
root Optional[str]

Optional root directory for the build context.

 None
dockerignore_file Optional[str]

Optional path to a dockerignore file. If not given, a file called .dockerignore in the build context root directory will be used instead if it exists.

 None
Source code in zenml/image_builders/build_context.py 
def __init__(
    self,
    root: Optional[str] = None,
    dockerignore_file: Optional[str] = None,
) -> None:
    """Initializes a build context.

    Args:
        root: Optional root directory for the build context.
        dockerignore_file: Optional path to a dockerignore file. If not
            given, a file called `.dockerignore` in the build context root
            directory will be used instead if it exists.
    """
    self._root = root
    self._dockerignore_file = dockerignore_file
    self._extra_files: Dict[str, str] = {}
add_directory(self, source, destination)

Adds a directory to the build context.

Parameters:

Name Type Description Default
source str

Path to the directory.

 required
destination str

The path inside the build context where the directory should be added.

 required

Exceptions:

Type Description
ValueError

If source does not point to a directory.
Source code in zenml/image_builders/build_context.py 
def add_directory(self, source: str, destination: str) -> None:
    """Adds a directory to the build context.

    Args:
        source: Path to the directory.
        destination: The path inside the build context where the directory
            should be added.

    Raises:
        ValueError: If `source` does not point to a directory.
    """
    if not fileio.isdir(source):
        raise ValueError(
            f"Can't add directory {source} to the build context as it "
            "does not exist or is not a directory."
        )

    for dir, _, files in fileio.walk(source):
        dir_path = Path(fileio.convert_to_str(dir))
        for file_name in files:
            file_name = fileio.convert_to_str(file_name)
            file_source = dir_path / file_name
            file_destination = (
                Path(destination)
                / dir_path.relative_to(source)
                / file_name
            )

            with file_source.open("r") as f:
                self._extra_files[file_destination.as_posix()] = f.read()
add_file(self, source, destination)

Adds a file to the build context.

Parameters:

Name Type Description Default
source str

The source of the file to add. This can either be a path or the file content.

 required
destination str

The path inside the build context where the file should be added.

 required
Source code in zenml/image_builders/build_context.py 
def add_file(self, source: str, destination: str) -> None:
    """Adds a file to the build context.

    Args:
        source: The source of the file to add. This can either be a path
            or the file content.
        destination: The path inside the build context where the file
            should be added.
    """
    if fileio.exists(source):
        with fileio.open(source) as f:
            self._extra_files[destination] = f.read()
    else:
        self._extra_files[destination] = source
write_archive(self, output_file, gzip=True)

Writes an archive of the build context to the given file.

Parameters:

Name Type Description Default
output_file IO[bytes]

The file to write the archive to.

 required
gzip bool

Whether to use gzip to compress the file.

 True
Source code in zenml/image_builders/build_context.py 
def write_archive(self, output_file: IO[bytes], gzip: bool = True) -> None:
    """Writes an archive of the build context to the given file.

    Args:
        output_file: The file to write the archive to.
        gzip: Whether to use `gzip` to compress the file.
    """
    from docker.utils import build as docker_build_utils

    files = self._get_files()
    extra_files = self._get_extra_files()

    context_archive = docker_build_utils.create_archive(
        fileobj=output_file,
        root=self._root,
        files=sorted(files),
        gzip=gzip,
        extra_files=extra_files,
    )

    build_context_size = os.path.getsize(context_archive.name)
    if (
        self._root
        and build_context_size > 50 * 1024 * 1024
        and not self.dockerignore_file
    ):
        # The build context exceeds 50MiB and we didn't find any excludes
        # in dockerignore files -> remind to specify a .dockerignore file
        logger.warning(
            "Build context size for docker image: `%s`. If you believe this is "
            "unreasonably large, make sure to include a `.dockerignore` file "
            "at the root of your build context `%s` or specify a custom file "
            "in the Docker configuration when defining your pipeline.",
            string_utils.get_human_readable_filesize(build_context_size),
            os.path.join(self._root, ".dockerignore"),
        )

local_image_builder

Local Docker image builder implementation.

LocalImageBuilder (BaseImageBuilder)

Local image builder implementation.

Source code in zenml/image_builders/local_image_builder.py 
class LocalImageBuilder(BaseImageBuilder):
    """Local image builder implementation."""

    @property
    def config(self) -> LocalImageBuilderConfig:
        """The stack component configuration.

        Returns:
            The configuration.
        """
        return cast(LocalImageBuilderConfig, self._config)

    @staticmethod
    def _check_prerequisites() -> None:
        """Checks that all prerequisites are installed.

        Raises:
            RuntimeError: If any of the prerequisites are not installed or
                running.
        """
        if not shutil.which("docker"):
            raise RuntimeError(
                "`docker` is required to run the local image builder."
            )

        if not docker_utils.check_docker():
            raise RuntimeError(
                "Unable to connect to the Docker daemon. There are two "
                "common causes for this:\n"
                "1) The Docker daemon isn't running.\n"
                "2) The Docker client isn't configured correctly. The client "
                "loads its configuration from the following file: "
                "$HOME/.docker/config.json. If your configuration file is in a "
                "different location, set it using the `DOCKER_CONFIG` "
                "environment variable."
            )

    def build(
        self,
        image_name: str,
        build_context: "BuildContext",
        docker_build_options: Optional[Dict[str, Any]] = None,
        container_registry: Optional["BaseContainerRegistry"] = None,
    ) -> str:
        """Builds and optionally pushes an image using the local Docker client.

        Args:
            image_name: Name of the image to build and push.
            build_context: The build context to use for the image.
            docker_build_options: Docker build options.
            container_registry: Optional container registry to push to.

        Returns:
            The Docker image repo digest.
        """
        self._check_prerequisites()

        docker_client = DockerClient.from_env()

        with tempfile.TemporaryFile(mode="w+b") as f:
            build_context.write_archive(f)

            # We use the client api directly here, so we can stream the logs
            output_stream = docker_client.images.client.api.build(
                fileobj=f,
                custom_context=True,
                tag=image_name,
                **(docker_build_options or {}),
            )
        docker_utils._process_stream(output_stream)

        if container_registry:
            return container_registry.push_image(image_name)
        else:
            return image_name
config: LocalImageBuilderConfig property readonly

The stack component configuration.

Returns:

Type Description
LocalImageBuilderConfig

The configuration.
build(self, image_name, build_context, docker_build_options=None, container_registry=None)

Builds and optionally pushes an image using the local Docker client.

Parameters:

Name Type Description Default
image_name str

Name of the image to build and push.

 required
build_context BuildContext

The build context to use for the image.

 required
docker_build_options Optional[Dict[str, Any]]

Docker build options.

 None
container_registry Optional[BaseContainerRegistry]

Optional container registry to push to.

 None

Returns:

Type Description
str

The Docker image repo digest.
Source code in zenml/image_builders/local_image_builder.py 
def build(
    self,
    image_name: str,
    build_context: "BuildContext",
    docker_build_options: Optional[Dict[str, Any]] = None,
    container_registry: Optional["BaseContainerRegistry"] = None,
) -> str:
    """Builds and optionally pushes an image using the local Docker client.

    Args:
        image_name: Name of the image to build and push.
        build_context: The build context to use for the image.
        docker_build_options: Docker build options.
        container_registry: Optional container registry to push to.

    Returns:
        The Docker image repo digest.
    """
    self._check_prerequisites()

    docker_client = DockerClient.from_env()

    with tempfile.TemporaryFile(mode="w+b") as f:
        build_context.write_archive(f)

        # We use the client api directly here, so we can stream the logs
        output_stream = docker_client.images.client.api.build(
            fileobj=f,
            custom_context=True,
            tag=image_name,
            **(docker_build_options or {}),
        )
    docker_utils._process_stream(output_stream)

    if container_registry:
        return container_registry.push_image(image_name)
    else:
        return image_name

LocalImageBuilderConfig (BaseImageBuilderConfig) pydantic-model

Local image builder configuration.

Source code in zenml/image_builders/local_image_builder.py 
class LocalImageBuilderConfig(BaseImageBuilderConfig):
    """Local image builder configuration."""

    @property
    def is_local(self) -> bool:
        """Checks if this stack component is running locally.

        Returns:
            True.
        """
        return True
is_local: bool property readonly

Checks if this stack component is running locally.

Returns:

Type Description
bool

True.

LocalImageBuilderFlavor (BaseImageBuilderFlavor)

Local image builder flavor.

Source code in zenml/image_builders/local_image_builder.py 
class LocalImageBuilderFlavor(BaseImageBuilderFlavor):
    """Local image builder flavor."""

    @property
    def name(self) -> str:
        """The flavor name.

        Returns:
            The flavor name.
        """
        return "local"

    @property
    def config_class(self) -> Type[LocalImageBuilderConfig]:
        """Config class.

        Returns:
            The config class.
        """
        return LocalImageBuilderConfig

    @property
    def implementation_class(self) -> Type[LocalImageBuilder]:
        """Implementation class.

        Returns:
            The implementation class.
        """
        return LocalImageBuilder
config_class: Type[zenml.image_builders.local_image_builder.LocalImageBuilderConfig] property readonly

Config class.

Returns:

Type Description
Type[zenml.image_builders.local_image_builder.LocalImageBuilderConfig]

The config class.
implementation_class: Type[zenml.image_builders.local_image_builder.LocalImageBuilder] property readonly

Implementation class.

Returns:

Type Description
Type[zenml.image_builders.local_image_builder.LocalImageBuilder]

The implementation class.
name: str property readonly

The flavor name.

Returns:

Type Description
str

The flavor name.