diff --git a/template/templates/Containerfile.j2 b/template/templates/Containerfile.j2
index 2ca58d7..390ca73 100644
--- a/template/templates/Containerfile.j2
+++ b/template/templates/Containerfile.j2
@@ -2,14 +2,6 @@
 
 FROM {{ recipe.base_image }}:{{ recipe.image_version }}
 
-LABEL {{ blue_build_utils::constants::BUILD_ID_LABEL }}="{{ build_id }}"
-LABEL org.opencontainers.image.title="{{ recipe.name }}"
-LABEL org.opencontainers.image.description="{{ recipe.description }}"
-{%- if let Some(repo) = self::get_repo_url() %}
-LABEL org.opencontainers.image.source="{{ repo }}"
-{%- endif %}
-LABEL io.artifacthub.package.readme-url=https://raw.githubusercontent.com/blue-build/cli/main/README.md
-
 ARG RECIPE={{ recipe_path.display() }}
 ARG IMAGE_REGISTRY={{ registry }}
 
@@ -19,3 +11,12 @@ ARG IMAGE_NAME="{{ recipe.name }}"
 ARG BASE_IMAGE="{{ recipe.base_image }}"
 
 {% include "modules/modules.j2" %}
+
+# Labels are added last since they cause cache misses with buildah
+LABEL {{ blue_build_utils::constants::BUILD_ID_LABEL }}="{{ build_id }}"
+LABEL org.opencontainers.image.title="{{ recipe.name }}"
+LABEL org.opencontainers.image.description="{{ recipe.description }}"
+{%- if let Some(repo) = self::get_repo_url() %}
+LABEL org.opencontainers.image.source="{{ repo }}"
+{%- endif %}
+LABEL io.artifacthub.package.readme-url=https://raw.githubusercontent.com/blue-build/cli/main/README.md