[NADE] Enable Code Generation by default in Snow App Bundle (#1167)

RELEASE-NOTES.md

@@ -21,6 +21,7 @@
 
 ## New additions
 * Added `snow app bundle` command that prepares a local folder in the project directory with artifacts to be uploaded to a stage as part of creating a Snowflake Native App.
+  * Snowflake Native App projects can now optionally generate CREATE FUNCTION or CREATE PROCEDURE declarations in setup scripts from Snowpark python code that includes decorators (e.g. @sproc, @udf).
 * Added `snow app validate` command that validates the setup script SQL used to create a Snowflake Native App for syntax validity, invalid object references, and best practices
   * Added new `native_app.scratch_stage` field to `snowflake.yml` schema to allow customizing the stage that the CLI uses to run the validation
 * Changed `snow app deploy` and `snow app run` to trigger validation of the uploaded setup script SQL and block uploads on validation failure, pass `--no-validate` to disable

src/snowflake/cli/plugins/nativeapp/codegen/snowpark/callback_source.py.jinja

@@ -2,7 +2,7 @@ import contextlib
 import functools
 import inspect
 import sys
-from typing import Callable
+from typing import Callable, Tuple
 
 try:
     import snowflake.snowpark
@@ -101,8 +101,7 @@ def __snowflake_internal_create_extension_fn_registration_callback():
 
 
     def __snowflake_internal_extension_fn_to_json(extension_fn):
-        if not isinstance(extension_fn.func, Callable):
-            # Unsupported case: extension function is a tuple
+        if not (isinstance(extension_fn.func, Callable) or isinstance(extension_fn.func, Tuple)):
             return
 
         if extension_fn.anonymous:
@@ -141,7 +140,8 @@ def __snowflake_internal_create_extension_fn_registration_callback():
         collected_extension_fn_json_list, extension_function_properties
     ):
         extension_fn_json = __snowflake_internal_extension_fn_to_json(extension_function_properties)
-        collected_extension_fn_json_list.append(extension_fn_json)
+        if extension_fn_json: # Do not append if extension_fn_json is None
+            collected_extension_fn_json_list.append(extension_fn_json)
         return False
 
     return functools.partial(

src/snowflake/cli/plugins/nativeapp/codegen/snowpark/python_processor.py

@@ -211,6 +211,7 @@ def process(
                 py_file=py_file,
             )
             collected_sql_files.append(sql_file)
+            insert_newline = False
             for extension_fn in extension_fns:
                 create_stmt = generate_create_sql_ddl_statement(extension_fn)
                 if create_stmt is None:
@@ -223,6 +224,9 @@ def process(
                     collected_output.append(grant_statements)
 
                 with open(sql_file, "a") as file:
+                    if insert_newline:
+                        file.write("\n")
+                    insert_newline = True
                     file.write(
                         f"-- Generated by the Snowflake CLI from {relative_py_file}\n"
                     )
@@ -343,8 +347,6 @@ def collect_extension_functions(
             )
 
             if collected_extension_function_json is None:
-                cc.warning(f"Error processing extension functions in {src_file}")
-                cc.warning("Skipping generating code of all objects from this file.")
                 continue
 
             collected_extension_functions = []
@@ -461,15 +463,20 @@ def generate_grant_sql_ddl_statements(
 
     if not extension_fn.application_roles:
         cc.warning(
-            "Skipping generation of 'GRANT USAGE ON ...' SQL statement for this object due to lack of application roles."
+            f"Skipping generation of 'GRANT USAGE ON ...' SQL statement for {extension_fn.function_type.upper()} {extension_fn.handler} due to lack of application roles."
         )
         return None
 
     grant_sql_statements = []
+    object_type = (
+        "PROCEDURE"
+        if extension_fn.function_type == ExtensionFunctionTypeEnum.PROCEDURE
+        else "FUNCTION"
+    )
     for app_role in extension_fn.application_roles:
         grant_sql_statement = dedent(
             f"""\
-            GRANT USAGE ON {get_sql_object_type(extension_fn)} {get_qualified_object_name(extension_fn)}({get_function_type_signature_for_grant(extension_fn)})
+            GRANT USAGE ON {object_type} {get_qualified_object_name(extension_fn)}({get_function_type_signature_for_grant(extension_fn)})
             TO APPLICATION ROLE {app_role};
             """
         ).strip()

src/snowflake/cli/plugins/nativeapp/manager.py

@@ -68,7 +68,6 @@
     SetupScriptFailedValidation,
     UnexpectedOwnerError,
 )
-from snowflake.cli.plugins.nativeapp.feature_flags import FeatureFlag
 from snowflake.cli.plugins.nativeapp.utils import verify_exists, verify_no_directories
 from snowflake.cli.plugins.stage.diff import (
     DiffResult,
@@ -343,14 +342,13 @@ def build_bundle(self) -> BundleMap:
         Populates the local deploy root from artifact sources.
         """
         mapped_files = build_bundle(self.project_root, self.deploy_root, self.artifacts)
-        if FeatureFlag.ENABLE_SETUP_SCRIPT_GENERATION.is_enabled():
-            compiler = NativeAppCompiler(
-                project_definition=self._project_definition,
-                project_root=self.project_root,
-                deploy_root=self.deploy_root,
-                generated_root=self.generated_root,
-            )
-            compiler.compile_artifacts()
+        compiler = NativeAppCompiler(
+            project_definition=self._project_definition,
+            project_root=self.project_root,
+            deploy_root=self.deploy_root,
+            generated_root=self.generated_root,
+        )
+        compiler.compile_artifacts()
         return mapped_files
 
     def sync_deploy_root_with_stage(

tests_e2e/__snapshots__/test_nativeapp.ambr

@@ -0,0 +1,102 @@
+# serializer version: 1
+# name: test_full_lifecycle_with_codegen
+  dict_values(['echo_proc: test'])
+# ---
+# name: test_full_lifecycle_with_codegen.1
+  dict_values(['echo_fn: test'])
+# ---
+# name: test_full_lifecycle_with_codegen.10
+  dict_values([10])
+# ---
+# name: test_full_lifecycle_with_codegen.2
+  dict_values(['echo_fn: infile: test'])
+# ---
+# name: test_full_lifecycle_with_codegen.3
+  dict_values(['echo_fn: test'])
+# ---
+# name: test_full_lifecycle_with_codegen.4
+  dict_values(['echo_fn: test'])
+# ---
+# name: test_full_lifecycle_with_codegen.5
+  dict_values([3])
+# ---
+# name: test_full_lifecycle_with_codegen.6
+  dict_values([10])
+# ---
+# name: test_full_lifecycle_with_codegen.7
+  '''
+  [
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      }
+  ]
+
+  '''
+# ---
+# name: test_full_lifecycle_with_codegen.8
+  dict_values(['echo_fn: infile: test'])
+# ---
+# name: test_full_lifecycle_with_codegen.9
+  '''
+  [
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      }
+  ]
+
+  '''
+# ---

tests_e2e/conftest.py

@@ -13,8 +13,10 @@
 # limitations under the License.
 
 import os
+import shutil
 import subprocess
 import tempfile
+from contextlib import contextmanager
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -102,3 +104,16 @@ def _install_snowcli_with_external_plugin(
 
 def _python_path(venv_path: Path) -> Path:
     return venv_path / "bin" / "python"
+
+
+# Inspired by project_directory fixture in tests_integration/conftest.py
+# This is a simpler implementation of that fixture, i.e. does not include supporting local PDFs.
+@pytest.fixture
+def project_directory(temp_dir, test_root_path):
+    @contextmanager
+    def _temporary_project_directory(project_name):
+        test_data_file = test_root_path / "test_data" / project_name
+        shutil.copytree(test_data_file, temp_dir, dirs_exist_ok=True)
+        yield Path(temp_dir)
+
+    return _temporary_project_directory

tests_e2e/test_data/nativeapp/.gitignore

@@ -0,0 +1,4 @@
+snowflake.local.yml
+output/
+**/__pycache__/
+**/.pytest_cache/

tests_e2e/test_data/nativeapp/python/cli_gen/accepted.py

@@ -0,0 +1,80 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from snowflake.snowpark.functions import sproc, udf
+from snowflake.snowpark.types import IntegerType
+
+
+def helper_fn(data: str) -> str:
+    return "infile: " + data
+
+
+@udf(
+    name="echo_fn_1",
+    native_app_params={
+        "schema": "ext_code_schema",
+        "application_roles": ["app_instance_role"],
+    },
+)
+def echo_fn_1(echo: str) -> str:
+    return "echo_fn: " + helper_fn(echo)
+
+
+# UDF name is given, imports and packages are empty, native_app_params empty, should be in the final output
+@udf(
+    name="echo_fn_2",
+    native_app_params={
+        "schema": "ext_code_schema",
+        "application_roles": ["app_instance_role"],
+    },
+)
+def echo_fn_2(echo: str) -> str:
+    return "echo_fn: " + echo
+
+
+# Inconsequential UDF Params, should have no effect on the DDL, should be in the final output
+@udf(
+    name="echo_fn_4",
+    is_permanent=True,
+    stage_location="@some_stage",
+    replace=False,
+    if_not_exists=True,
+    session=None,
+    parallel=4,
+    max_batch_size=2,
+    statement_params={},
+    strict=True,
+    secure=True,
+    immutable=True,
+    comment="some comment",
+    native_app_params={
+        "schema": "ext_code_schema",
+        "application_roles": ["app_instance_role"],
+    },
+)
+def echo_fn_4(echo: str) -> str:
+    return "echo_fn: " + echo
+
+
+@sproc(
+    return_type=IntegerType(),
+    input_types=[IntegerType(), IntegerType()],
+    packages=["snowflake-snowpark-python"],
+    native_app_params={
+        "schema": "ext_code_schema",
+        "application_roles": ["app_instance_role"],
+    },
+)
+def add_sp(session_, x, y):
+    return x + y

tests_e2e/test_data/nativeapp/python/cli_gen/errors/e1.py

@@ -0,0 +1,28 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from snowflake.snowpark.functions import udf
+
+
+# Should be rejected by Snowpark since type hints are absent, sql should not be in the final output.
+@udf(name="echo_fn_2")
+def echo_fn_2(echo) -> str:
+    return "echo_fn: " + echo
+
+
+"""
+Warning message generated by the CLI on skipping this file:
+Could not fetch Snowpark objects from /Users/bgoel/snowcli/tests_e2e/test_data/nativeapp/python/cli_gen/errors/e1.py due to the following Snowpark-internal error:
+ An exception occurred while executing file:  the number of arguments (1) is different from the number of argument type hints (0)
+"""