rebased onto main with bundle changes, addressed PR feedback, test pa…

…sses when using real config.toml connection locally, pushing to test on Github now

fix code quality failure, loosen restriction on checking stderr when CLI's config.toml has insufficient permissions

src/snowflake/cli/plugins/nativeapp/codegen/snowpark/callback_source.py.jinja

@@ -102,7 +102,6 @@ def __snowflake_internal_create_extension_fn_registration_callback():
 
     def __snowflake_internal_extension_fn_to_json(extension_fn):
         if not (isinstance(extension_fn.func, Callable) or isinstance(extension_fn.func, Tuple)):
-            # Unsupported case: extension function is a tuple
             return
 
         if extension_fn.anonymous:

src/snowflake/cli/plugins/nativeapp/codegen/snowpark/python_processor.py

@@ -140,7 +140,7 @@ def _execute_in_sandbox(
 
     if completed_process.returncode != 0:
         cc.warning(
-            f"Could not fetch Snowpark objects from {py_file} due to the following Snowpark-internal error:\n {completed_process.stderr}"
+            f"Could not fetch Snowpark objects from {py_file} due to the following error:\n {completed_process.stderr}"
         )
         cc.warning("Continuing execution for the rest of the python files.")
         return None
@@ -211,6 +211,7 @@ def process(
                 py_file=py_file,
             )
             collected_sql_files.append(sql_file)
+            insert_newline = False
             for extension_fn in extension_fns:
                 create_stmt = generate_create_sql_ddl_statement(extension_fn)
                 if create_stmt is None:
@@ -223,8 +224,11 @@ def process(
                     collected_output.append(grant_statements)
 
                 with open(sql_file, "a") as file:
+                    if insert_newline:
+                        file.write("\n")
+                    insert_newline = True
                     file.write(
-                        f"\n-- Generated by the Snowflake CLI from {relative_py_file}\n"
+                        f"-- Generated by the Snowflake CLI from {relative_py_file}\n"
                     )
                     file.write(f"-- DO NOT EDIT\n")
                     file.write(create_stmt)

tests_e2e/__snapshots__/test_nativeapp.ambr

@@ -1,91 +1,102 @@
 # serializer version: 1
 # name: test_full_lifecycle_with_codegen
-  '''
-  call codegen_nativeapp_af3d785601654fa0a9a105883b626866_app.ext_code_schema.py_echo_proc('test')
-  +-----------------+
-  | PY_ECHO_PROC    |
-  |-----------------|
-  | echo_proc: test |
-  +-----------------+
-
-  '''
+  dict_values(['echo_proc: test'])
 # ---
 # name: test_full_lifecycle_with_codegen.1
-  '''
-  select codegen_nativeapp_af3d785601654fa0a9a105883b626866_app.ext_code_schema.py_echo_fn('test')
-  +------------------------------------------------------------------------------+
-  | CODEGEN_NATIVEAPP_AF3D785601654FA0A9A105883B626866_APP.EXT_CODE_SCHEMA.PY_EC |
-  | HO_FN('TEST')                                                                |
-  |------------------------------------------------------------------------------|
-  | echo_fn: test                                                                |
-  +------------------------------------------------------------------------------+
-
-  '''
+  dict_values(['echo_fn: test'])
+# ---
+# name: test_full_lifecycle_with_codegen.10
+  dict_values([10])
 # ---
 # name: test_full_lifecycle_with_codegen.2
-  '''
-  select codegen_nativeapp_af3d785601654fa0a9a105883b626866_app.ext_code_schema.echo_fn_2('test')
-  +------------------------------------------------------------------------------+
-  | CODEGEN_NATIVEAPP_AF3D785601654FA0A9A105883B626866_APP.EXT_CODE_SCHEMA.ECHO_ |
-  | FN_2('TEST')                                                                 |
-  |------------------------------------------------------------------------------|
-  | echo_fn: test                                                                |
-  +------------------------------------------------------------------------------+
-
-  '''
+  dict_values(['echo_fn: infile: test'])
 # ---
 # name: test_full_lifecycle_with_codegen.3
-  '''
-  select codegen_nativeapp_af3d785601654fa0a9a105883b626866_app.ext_code_schema.echo_fn_4('test')
-  +------------------------------------------------------------------------------+
-  | CODEGEN_NATIVEAPP_AF3D785601654FA0A9A105883B626866_APP.EXT_CODE_SCHEMA.ECHO_ |
-  | FN_4('TEST')                                                                 |
-  |------------------------------------------------------------------------------|
-  | echo_fn: test                                                                |
-  +------------------------------------------------------------------------------+
-
-  '''
+  dict_values(['echo_fn: test'])
 # ---
 # name: test_full_lifecycle_with_codegen.4
-  '''
-  call codegen_nativeapp_af3d785601654fa0a9a105883b626866_app.ext_code_schema.add_sp(1, 2)
-  +--------+
-  | ADD_SP |
-  |--------|
-  | 3      |
-  +--------+
-
-  '''
+  dict_values(['echo_fn: test'])
 # ---
 # name: test_full_lifecycle_with_codegen.5
+  dict_values([3])
+# ---
+# name: test_full_lifecycle_with_codegen.6
+  dict_values([10])
+# ---
+# name: test_full_lifecycle_with_codegen.7
   '''
-  select codegen_nativeapp_af3d785601654fa0a9a105883b626866_app.ext_code_schema.sum_int_dec(10)
-  +------------------------------------------------------------------------------+
-  | CODEGEN_NATIVEAPP_AF3D785601654FA0A9A105883B626866_APP.EXT_CODE_SCHEMA.SUM_I |
-  | NT_DEC(10)                                                                   |
-  |------------------------------------------------------------------------------|
-  | 10                                                                           |
-  +------------------------------------------------------------------------------+
+  [
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      }
+  ]
 
   '''
 # ---
-# name: test_full_lifecycle_with_codegen.6
+# name: test_full_lifecycle_with_codegen.8
+  dict_values(['echo_fn: infile: test'])
+# ---
+# name: test_full_lifecycle_with_codegen.9
   '''
-  select * from TABLE(codegen_nativeapp_af3d785601654fa0a9a105883b626866_app.ext_code_schema.alt_int(10))
-  +--------+
-  | NUMBER |
-  |--------|
-  | 1      |
-  | -1     |
-  | 1      |
-  | -1     |
-  | 1      |
-  | -1     |
-  | 1      |
-  | -1     |
-  | 1      |
-  | -1     |
-  +--------+
+  [
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      },
+      {
+          "NUMBER": 1
+      },
+      {
+          "NUMBER": -1
+      }
+  ]
 
   '''
 # ---

tests_e2e/test_data/nativeapp/python/cli_gen/errors/e1.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from snowflake.snowpark.functions import udf
 
 

tests_e2e/test_data/nativeapp/python/cli_gen/ignored/i1.py

@@ -1,3 +1,17 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 from snowflake.snowpark.functions import udf
 
 # Should be ignored at callback, will process file but not generate any sql

tests_e2e/test_data/nativeapp/python/user_gen/echo.py

@@ -1,3 +1,18 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
 def echo_fn(data):
     return "echo_fn: " + data
 

tests_e2e/test_data/nativeapp/resources/unused.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2024 Snowflake Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests_e2e/test_data/nativeapp/root_files/README.md

@@ -1 +1,17 @@
+<!--
+ Copyright (c) 2024 Snowflake Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
 DUMMY README FILE

tests_e2e/test_data/nativeapp/root_files/setup_scripts/setup.sql

@@ -1,3 +1,19 @@
+/*
+ Copyright (c) 2024 Snowflake Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
 create or replace application role app_instance_role;
 
 create or alter versioned schema ext_code_schema;

tests_e2e/test_data/nativeapp/root_files/setup_scripts/user_gen_procs.sql

@@ -1,3 +1,19 @@
+/*
+ Copyright (c) 2024 Snowflake Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
 -- user wraps a method from a python file into a procedure
 create or replace procedure ext_code_schema.py_echo_proc(STR string)
   RETURNS STRING

tests_e2e/test_data/nativeapp/root_files/setup_scripts/user_gen_udf.sql

@@ -1,3 +1,19 @@
+/*
+ Copyright (c) 2024 Snowflake Inc.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
 -- user wraps a method from a python file into a function
 create or replace function ext_code_schema.py_echo_fn(STR string)
   RETURNS STRING