Coolstore evaluation

Makefile

@@ -8,9 +8,10 @@ DEMO_MODE ?= False
 DROP_TABLES ?= False
 HUB_URL ?= ""
 IMPORTER_ARGS ?= ""
+POSTGRES_RUN_ARGS ?= 
 
 run-postgres:
-	$(CONTAINER_RUNTIME) run -it -v data:/var/lib/postgresql/data -e POSTGRES_USER=kai -e POSTGRES_PASSWORD=dog8code -e POSTGRES_DB=kai -p 5432:5432 docker.io/library/postgres:16.3
+	$(CONTAINER_RUNTIME) run -it $(POSTGRES_RUN_ARGS) -v data:/var/lib/postgresql/data -e POSTGRES_USER=kai -e POSTGRES_PASSWORD=dog8code -e POSTGRES_DB=kai -p 5432:5432 docker.io/library/postgres:16.3
 
 run-server:
 	PYTHONPATH=$(KAI_PYTHON_PATH) LOGLEVEL=$(LOGLEVEL) DEMO_MODE=$(DEMO_MODE) gunicorn --timeout 3600 -w $(NUM_WORKERS) --bind localhost:8080 --worker-class aiohttp.GunicornWebWorker 'kai.server:app()'

notebooks/evaluation/.gitignore

@@ -0,0 +1 @@
+data/

notebooks/evaluation/templates/evaluate.jinja

@@ -0,0 +1,51 @@
+<<SYS>>
+You are an expert in judging source code that is migrated from JavaEE to Quarkus.
+You will be given a JavaEE file and a list of issues identified in it which need to be fixed to migrate the file to Quarkus. These issues contain description about the exact change that needs to happen.
+You will also be given an updated file which is supposed to fix the issues identified in the original JavaEE file.
+Your job is to rate the migrated Quarkus file based on the following criteria.
+
+## Criteria:
+1. Fixing the issue:
+  * Ensure that all of the original issues pointed out in the JavaEE file are fixed in the updated file.
+  * If an issue is not fixed exactly as explained in the description of the issue, do not consider it fixed.
+  * Do not consider an alternative approach as a valid fix.
+  * If none of the issues are fixed, rate the lowest.
+  * If only some of the issues are fixed, give a rating proportional to the number of issues fixed.
+  * If all of the issues are fixed, rate the highest.
+2. General syntactical issues:
+  * Ensure that the updated file is syntactically valid and can be compiled. 
+  * Ensure that all classes, interfaces, annotations, etc used in the file are valid and imported correctly.
+  * Make sure that fixes do not change the original functionality.
+  * If you find any of the above issues, rate the file lower proportional to the number of issues you find. 
+  * If the file doesn't have issues, rating must be higher.
+
+Each metric defined in the criteria above must be rated on a scale of 0 to 5 where 0 is the lowest and 5 is the highest. The rating should be proportional to issues you find for the each metric.
+Output only the rating for each metric on a newline. Also include reasoning on a line following the rating.
+
+Produce your output exactly as per following format:
+
+// rating here on the first line. Only output a numeric rating. Do not output any text on this line.
+// reasoning here on the second line. Only output text in the reasoning. No numbers here.
+<</SYS>>
+[INST]
+
+## JavaEE file:
+
+```java
+{{ original_file }}
+```
+
+## Issues identified in the JavaEE file:
+
+{% for incident in incidents %}
+### incident {{ loop.index0 }}
+Line number of the issue to fix: {{ incident.line_number }}
+Expected fix: {{ incident.message }}
+{% endfor %}
+
+## Updated file:
+
+```java
+{{ updated_file }}
+```
+[/INST]

notebooks/evaluation/templates/few_shot/default.jinja

@@ -0,0 +1,58 @@
+{% if model_provider.llama_header %}<s>[INST]You are an AI Assistant trained on migrating enterprise JavaEE code to Quarkus.<<SYS>>{% endif %}
+I will give you a JavaEE file for which I want to take one step towards migrating to Quarkus.
+
+I will provide you with static source code analysis information highlighting an issue which needs to be addressed.
+
+I will also provide you with solved examples of how a similar issue was solved in the past.
+
+You can refer to the solved example for a pattern of how to update the input Java EE file to Quarkus.
+
+Fix only the problem described. Other problems will be solved in subsequent steps so it is unnecessary to handle them now.
+
+Before attempting to migrate the code to Quarkus, briefly reason through what changes are required and why.
+
+Pay attention to changes you make to imports we need to consider.
+
+Remember when updating or adding annotations that the class must be imported.
+
+After you have shared your step by step thinking, provide a full output of the updated file.
+
+# Input information
+
+## Input File
+
+File name: "{{ src_file_name }}"
+Source file contents:
+```{{ src_file_language }}
+{{ src_file_contents | safe }}
+```
+
+## Issues
+
+{% for incident in incidents %}
+### incident {{ loop.index0 }}
+Line number: {{ incident.line_number }}
+Message: {{ incident.message }}
+{% if (incidents|length == 1 or loop.index0 == 6) and incident.solution_str is defined %}
+{{ incident.solution_str | safe }}
+{% endif %}
+{% endfor %}
+
+# Output Instructions
+Structure your output exactly in Markdown format such as:
+
+## Reasoning
+Write the step by step reasoning in this markdown section. Do not include code in this section, please only include text.
+
+## Updated File
+```java
+// Write the updated file for Quarkus in this section. If the file should be removed, make the content of the updated file a comment explaining it should be removed.
+```
+
+Make sure your output is exactly in the format described above. Pay special attention to markdown sections.
+
+{% if model_provider.model_id == 'mistralai/mixtral-8x7b-instruct-v01' %}
+Only use ## to begin section headers in the markdown above. Output the entire file. 
+{% endif %}
+
+{% if model_provider.llama_header %}[/INST]{% endif %}

notebooks/evaluation/templates/few_shot/easy_hardcoded.jinja

@@ -0,0 +1,64 @@
+{% if model_provider.llama_header %}<s>[INST]You are an AI Assistant trained on migrating enterprise JavaEE code to Quarkus.<<SYS>>{% endif %}
+I will give you a JavaEE file for which I want to take one step towards migrating to Quarkus.
+
+I will provide you with static source code analysis information highlighting an issue which needs to be addressed.
+
+I will also provide you with a solved example of how a similar issue was solved in the past.
+
+You can refer to the solved example for a pattern of how to update the input Java EE file to Quarkus.
+
+Fix only the problem described. Other problems will be solved in subsequent steps so it is unnecessary to handle them now.
+
+Before attempting to migrate the code to Quarkus, briefly reason through what changes are required and why.
+
+Pay attention to changes you make to imports we need to consider.
+
+Remember when updating or adding annotations that the class must be imported.
+
+After you have shared your step by step thinking, provide a full output of the updated file.
+
+# Input information
+
+## Input File
+
+File name: "{{ src_file_name }}"
+Source file contents:
+```{{ src_file_language }}
+{{ src_file_contents | safe }}
+```
+
+## Issues
+
+{% for incident in incidents %}
+### incident {{ loop.index0 }}
+Line number: {{ incident.line_number }}
+{% endfor %}
+Solution diff:
+1c1
+< import javax.enterprise.context.Dependent;
+---
+> import jakarta.enterprise.context.Dependent;
+
+
+# Output Instructions
+Structure your output exactly in Markdown format such as:
+
+## Reasoning
+Write the step by step reasoning in this markdown section. Do not include code in this section, please only include text.
+
+## Updated File
+```java
+// Write the updated file for Quarkus in this section. If the file should be removed, make the content of the updated file a comment explaining it should be removed.
+```
+
+## Additional Information (optional)
+
+If you have any additional details or steps that need to be performed, put it here.
+
+Make sure your output is exactly in the format described above. Pay special attention to markdown sections.
+
+{% if model_provider.model_id == 'mistralai/mixtral-8x7b-instruct-v01' %}
+Only use ## to begin section headers in the markdown above. Output the entire file. 
+{% endif %}
+
+{% if model_provider.llama_header %}[/INST]{% endif %}

notebooks/evaluation/templates/few_shot/easy_hardcoded_summary.jinja

@@ -0,0 +1,171 @@
+{% if model_provider.llama_header %}<s>[INST]You are an AI Assistant trained on migrating enterprise JavaEE code to Quarkus.<<SYS>>{% endif %}
+I will give you a JavaEE file for which I want to take one step towards migrating to Quarkus.
+
+I will provide you with static source code analysis information highlighting an issue which needs to be addressed.
+
+Wherever available, I will also provide you with a summary of a solution for a similar issue solved in the past.
+
+You can refer to the solved example for a pattern of how to update the input Java EE file to Quarkus.
+
+Fix the problem described.
+
+Before attempting to migrate the code to Quarkus, briefly reason through what changes are required and why.
+
+Pay attention to changes you make to imports we need to consider.
+
+Remember when updating or adding annotations that the class must be imported.
+
+After you have shared your step by step thinking, provide a full output of the updated file. 
+
+# Input information
+
+## Input File
+
+File name: "ShoppingCart.java"
+Source file contents:
+```java
+package com.redhat.coolstore.model;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.enterprise.context.Dependent;
+
+@Dependent
+public class ShoppingCart implements Serializable {
+
+	private static final long serialVersionUID = -1108043957592113528L;
+
+	private double cartItemTotal;
+
+	private double cartItemPromoSavings;
+
+	private double shippingTotal;
+
+	private double shippingPromoSavings;
+
+	private double cartTotal;
+
+	private List<ShoppingCartItem> shoppingCartItemList = new ArrayList<ShoppingCartItem>();
+
+	public ShoppingCart() {
+
+	}
+
+	public List<ShoppingCartItem> getShoppingCartItemList() {
+		return shoppingCartItemList;
+	}
+
+	public void setShoppingCartItemList(List<ShoppingCartItem> shoppingCartItemList) {
+		this.shoppingCartItemList = shoppingCartItemList;
+	}
+
+	public void resetShoppingCartItemList() {
+		shoppingCartItemList = new ArrayList<ShoppingCartItem>();
+	}
+
+	public void addShoppingCartItem(ShoppingCartItem sci) {
+
+		if ( sci != null ) {
+
+			shoppingCartItemList.add(sci);
+
+		}
+
+	}
+
+	public boolean removeShoppingCartItem(ShoppingCartItem sci) {
+
+		boolean removed = false;
+
+		if ( sci != null ) {
+
+			removed = shoppingCartItemList.remove(sci);
+
+		}
+
+		return removed;
+
+	}
+
+	public double getCartItemTotal() {
+		return cartItemTotal;
+	}
+
+	public void setCartItemTotal(double cartItemTotal) {
+		this.cartItemTotal = cartItemTotal;
+	}
+
+	public double getShippingTotal() {
+		return shippingTotal;
+	}
+
+	public void setShippingTotal(double shippingTotal) {
+		this.shippingTotal = shippingTotal;
+	}
+
+	public double getCartTotal() {
+		return cartTotal;
+	}
+
+	public void setCartTotal(double cartTotal) {
+		this.cartTotal = cartTotal;
+	}
+
+	public double getCartItemPromoSavings() {
+		return cartItemPromoSavings;
+	}
+
+	public void setCartItemPromoSavings(double cartItemPromoSavings) {
+		this.cartItemPromoSavings = cartItemPromoSavings;
+	}
+
+	public double getShippingPromoSavings() {
+		return shippingPromoSavings;
+	}
+
+	public void setShippingPromoSavings(double shippingPromoSavings) {
+		this.shippingPromoSavings = shippingPromoSavings;
+	}
+
+	@Override
+	public String toString() {
+		return "ShoppingCart [cartItemTotal=" + cartItemTotal
+				+ ", cartItemPromoSavings=" + cartItemPromoSavings
+				+ ", shippingTotal=" + shippingTotal
+				+ ", shippingPromoSavings=" + shippingPromoSavings
+				+ ", cartTotal=" + cartTotal + ", shoppingCartItemList="
+				+ shoppingCartItemList + "]";
+	}
+}
+
+```
+
+## Issues
+
+### incident 0
+Line number: 7
+Message: Replace `javax.enterprise` with `jakarta.enterprise`
+Solution summary:
+The change made to address the migration issue is the replacement of the import statement javax.enterprise.context.Dependent with jakarta.enterprise.context.Dependent. This updates the code to be compatible with Quarkus, which uses the jakarta namespace instead of javax.
+
+# Output Instructions
+Structure your output exactly in Markdown format such as:
+
+## Reasoning
+Write the step by step reasoning in this markdown section. Do not include code in this section, please only include text.
+
+## Updated File
+```java
+// Write the full updated file for Quarkus in this section. If the file should be removed, make the content of the updated file a comment explaining it should be removed.
+```
+
+
+Make sure your output is exactly in the format described above. Pay special attention to markdown sections.
+
+{% if model_provider.model_id == 'mistralai/mixtral-8x7b-instruct-v01' %}
+Only use ## to begin section headers in the markdown above. Output the entire file. 
+{% endif %}
+
+{% if model_provider.llama_header %}[/INST]{% endif %}