SNOW-1643030 Convert package scripts to post-deploy hooks (#1538)

PDFv2 doesn't support package scripts, so we need to convert them to post-deploy hooks. Since package scripts use a different Jinja template syntax (`{{ }}` instead of `<% %>`), we need to convert the actual files as well.

src/snowflake/cli/api/entities/utils.py

@@ -303,27 +303,34 @@ def render_script_templates(
     - List of rendered scripts content
     Size of the return list is the same as the size of the input scripts list.
     """
-    scripts_contents = []
-    for relpath in scripts:
-        script_full_path = SecurePath(project_root) / relpath
-        try:
-            template_content = script_full_path.read_text(file_size_limit_mb=UNLIMITED)
-            env = override_env or choose_sql_jinja_env_based_on_template_syntax(
-                template_content, reference_name=relpath
-            )
-            result = env.from_string(template_content).render(jinja_context)
-            scripts_contents.append(result)
+    return [
+        render_script_template(project_root, jinja_context, script, override_env)
+        for script in scripts
+    ]
 
-        except FileNotFoundError as e:
-            raise MissingScriptError(relpath) from e
 
-        except jinja2.TemplateSyntaxError as e:
-            raise InvalidTemplateInFileError(relpath, e, e.lineno) from e
+def render_script_template(
+    project_root: Path,
+    jinja_context: dict[str, Any],
+    script: str,
+    override_env: Optional[jinja2.Environment] = None,
+) -> str:
+    script_full_path = SecurePath(project_root) / script
+    try:
+        template_content = script_full_path.read_text(file_size_limit_mb=UNLIMITED)
+        env = override_env or choose_sql_jinja_env_based_on_template_syntax(
+            template_content, reference_name=script
+        )
+        return env.from_string(template_content).render(jinja_context)
+
+    except FileNotFoundError as e:
+        raise MissingScriptError(script) from e
 
-        except jinja2.UndefinedError as e:
-            raise InvalidTemplateInFileError(relpath, e) from e
+    except jinja2.TemplateSyntaxError as e:
+        raise InvalidTemplateInFileError(script, e, e.lineno) from e
 
-    return scripts_contents
+    except jinja2.UndefinedError as e:
+        raise InvalidTemplateInFileError(script, e) from e
 
 
 def validation_item_to_str(item: dict[str, str | int]):

src/snowflake/cli/api/project/definition_conversion.py

@@ -12,9 +12,14 @@
 from snowflake.cli.api.constants import (
     DEFAULT_ENV_FILE,
     DEFAULT_PAGES_DIR,
+    PROJECT_TEMPLATE_VARIABLE_CLOSING,
     PROJECT_TEMPLATE_VARIABLE_OPENING,
     SNOWPARK_SHARED_MIXIN,
 )
+from snowflake.cli.api.entities.utils import render_script_template
+from snowflake.cli.api.project.schemas.entities.common import (
+    SqlScriptHookType,
+)
 from snowflake.cli.api.project.schemas.native_app.application import Application
 from snowflake.cli.api.project.schemas.native_app.native_app import NativeApp
 from snowflake.cli.api.project.schemas.native_app.package import Package
@@ -28,6 +33,7 @@
 )
 from snowflake.cli.api.project.schemas.snowpark.snowpark import Snowpark
 from snowflake.cli.api.project.schemas.streamlit.streamlit import Streamlit
+from snowflake.cli.api.rendering.jinja import get_basic_jinja_env
 
 log = logging.getLogger(__name__)
 
@@ -200,6 +206,24 @@ def _find_manifest():
         # which use POSIX paths as default values
         return manifest_path.as_posix()
 
+    def _convert_package_script_files(package_scripts: list[str]):
+        # PDFv2 doesn't support package scripts, only post-deploy scripts, so we
+        # need to convert the Jinja syntax from {{ }} to <% %>
+        # Luckily, package scripts only support {{ package_name }}, so let's convert that tag
+        # to v2 template syntax by running it though the template process with a fake
+        # package name that's actually a valid v2 template, which will be evaluated
+        # when the script is used as a post-deploy script
+        fake_package_replacement_template = f"{PROJECT_TEMPLATE_VARIABLE_OPENING} ctx.entities.{package_entity_name}.identifier {PROJECT_TEMPLATE_VARIABLE_CLOSING}"
+        jinja_context = dict(package_name=fake_package_replacement_template)
+        post_deploy_hooks = []
+        for script_file in package_scripts:
+            new_contents = render_script_template(
+                project_root, jinja_context, script_file, get_basic_jinja_env()
+            )
+            (project_root / script_file).write_text(new_contents)
+            post_deploy_hooks.append(SqlScriptHookType(sql_script=script_file))
+        return post_deploy_hooks
+
     package_entity_name = "pkg"
     package = {
         "type": "application package",
@@ -218,9 +242,16 @@ def _find_manifest():
     }
     if native_app.package:
         package["distribution"] = native_app.package.distribution
-        if package_meta := _make_meta(native_app.package):
+        package_meta = _make_meta(native_app.package)
+        if native_app.package.scripts:
+            converted_post_deploy_hooks = _convert_package_script_files(
+                native_app.package.scripts
+            )
+            package_meta["post_deploy"] = (
+                package_meta.get("post_deploy", []) + converted_post_deploy_hooks
+            )
+        if package_meta:
             package["meta"] = package_meta
-        # todo migrate package scripts (requires migrating template tags)
 
     app_entity_name = "app"
     app = {
@@ -266,12 +297,6 @@ def _check_if_project_definition_meets_requirements(
         log.warning(
             "Your V1 definition contains templates. We cannot guarantee the correctness of the migration."
         )
-    if pd.native_app and pd.native_app.package and pd.native_app.package.scripts:
-        raise ClickException(
-            "Your project file contains a native app definition that uses package scripts. "
-            "Package scripts are not supported in definition version 2 and require manual conversion "
-            "to post-deploy scripts."
-        )
 
 
 def _process_streamlit_files(

tests/test_data/projects/migration_package_scripts/app/README.md

@@ -0,0 +1,4 @@
+# README
+
+This directory contains an extremely simple application that is used for
+integration testing SnowCLI.

tests/test_data/projects/migration_package_scripts/app/manifest.yml

@@ -0,0 +1,18 @@
+# This is a manifest.yml file, a required component of creating a native application.
+# This file defines properties required by the application package, including the location of the setup script and version definitions.
+# Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-manifest for a detailed understanding of this file.
+
+manifest_version: 1
+
+version:
+  name: dev
+  label: "Dev Version"
+  comment: "Default version used for development. Override for actual deployment."
+
+artifacts:
+  setup_script: setup.sql
+  readme: README.md
+
+configuration:
+  log_level: INFO
+  trace_level: ALWAYS

tests/test_data/projects/migration_package_scripts/app/setup.sql

@@ -0,0 +1,19 @@
+create application role if not exists app_public;
+create or alter versioned schema core;
+
+    create or replace procedure core.echo(inp varchar)
+    returns varchar
+    language sql
+    immutable
+    as
+    $$
+    begin
+        return inp;
+    end;
+    $$;
+
+    grant usage on procedure core.echo(varchar) to application role app_public;
+
+    create or replace view core.shared_view as select * from my_shared_content.shared_table;
+
+    grant select on view core.shared_view to application role app_public;

tests/test_data/projects/migration_package_scripts/package_scripts/001.sql

@@ -0,0 +1,2 @@
+-- Just a demo package script, won't actually be executed in tests
+select * from {{ package_name }}.my_schema.my_table

tests/test_data/projects/migration_package_scripts/package_scripts/002.sql

@@ -0,0 +1,2 @@
+-- Just a demo package script, won't actually be executed in tests
+select * from {{    package_name    }}.my_schema.my_table

tests/test_data/projects/migration_package_scripts/snowflake.yml

@@ -0,0 +1,11 @@
+definition_version: 1
+native_app:
+  name: myapp
+  source_stage: app_src.stage
+  artifacts:
+    - src: app/*
+      dest: ./
+  package:
+    scripts:
+      - package_scripts/001.sql
+      - package_scripts/002.sql

tests/workspace/test_manager.py

@@ -16,6 +16,7 @@
 import logging
 import os
 from pathlib import Path
+from textwrap import dedent
 from unittest import mock
 
 import pytest
@@ -128,6 +129,20 @@ def test_migration_native_app_no_artifacts(runner, project_directory):
     assert "Could not bundle Native App artifacts" in result.output
 
 
+def test_migration_native_app_package_scripts(runner, project_directory):
+    with project_directory("migration_package_scripts") as project_dir:
+        result = runner.invoke(["ws", "migrate"])
+        assert result.exit_code == 0
+        package_scripts_dir = project_dir / "package_scripts"
+        for file in package_scripts_dir.iterdir():
+            assert file.read_text() == dedent(
+                """\
+                -- Just a demo package script, won't actually be executed in tests
+                select * from <% ctx.entities.pkg.identifier %>.my_schema.my_table
+                """
+            )
+
+
 @pytest.mark.parametrize(
     "project_directory_name", ["snowpark_templated_v1", "streamlit_templated_v1"]
 )