From cddbbf844dce7ee2b179e738af1b8510fc7580fe Mon Sep 17 00:00:00 2001
From: leduc <leduc@MBP14.local>
Date: Sat, 15 Mar 2025 12:33:21 +0100
Subject: [PATCH] 70b analysis doc

---
 .DS_Store | Bin 6148 -> 6148 bytes
 README.md |   8 +++++++-
 main.py   |   8 ++++++--
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/.DS_Store b/.DS_Store
index 96bed275d6f891f5ed16243b0b39b8e09424fd1a..8fbf2d46547d34c47c7572ca7d64635fca6d138a 100644
GIT binary patch
delta 111
zcmZoMXffEJ#u9f|hJk^Bg+Y%YogtH<Bsbs1B`GIA2`I*Kn59qg%%NkBsPZXr`Lp5C
gBIxo38HT~h`MCu^Jq!$d0t}P$Sk*SKVu=z40A4N}hyVZp

delta 111
zcmZoMXffEJ#u9gG76StV3xgg*IzuKyNp8N2OHxjL5>SjoK-9$c^r2&psPZXz<qI+l
agOl@f3xIkU7<ex*OwMCf+q{Y;N*Dkk3LDb^

diff --git a/README.md b/README.md
index e436279..78e6e91 100644
--- a/README.md
+++ b/README.md
@@ -78,13 +78,19 @@ The tool currently tests models on these coding challenges:
 3. Initial syntax validation is performed
 4. Code that fails validation is passed to Together API for advanced code analysis
 5. Code that passes validation is executed and validated with given data and compared to expected results
-6. Each test is run 4 times for consistency and only the last 3 results are used for metrics
 
 ### Test Validation
 For each test case:
 - Input values are provided to the function
 - Output is compared with expected results
 - Test results are marked as ✅ (pass) or ❌ (fail)
+- The last 3 results of 4are used for metrics
+- An online 70b model (via free Together API) : meta-llama/Llama-3.3-70B-Instruct-Turbo-Free is used to give structured explanations for failed code
+
+### Metrics
+- Success Rate: Percentage of successful test cases
+- Tokens per Second (tk/sec): Number of tokens processed per second
+- Response Time: Time taken to generate a response
 
 Example test cases:
 ```plaintext
diff --git a/main.py b/main.py
index 1b02210..bc4f7fe 100755
--- a/main.py
+++ b/main.py
@@ -111,8 +111,12 @@ Model: {model}"""
         
         analysis = response.choices[0].message.content
         should_pass = "SHOULD PASS" in analysis.upper()
-        if verbose: print(f"\n{BLUE}[{model}] Together Analysis:{ENDC}")
-        if verbose: print(f"{GREEN if should_pass else RED}{analysis}{ENDC}")
+        if verbose:
+            print(f"\n{BLUE}{'='*50}{ENDC}")
+            print(f"{BLUE}[{model}] TOGETHER API CODE ANALYSIS:{ENDC}")
+            print(f"{BLUE}{'='*50}{ENDC}")
+            print(f"{GREEN if should_pass else RED}{analysis}{ENDC}")
+            print(f"{BLUE}{'='*50}{ENDC}")
         return should_pass
     except Exception as e:
         print(f"\n{RED}Error getting Together API analysis: {e}{ENDC}")