first commit

.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+.ipynb_checkpoints
+.env

README.md

@@ -0,0 +1,32 @@
+---
+title: Mock JSON Generator
+emoji: 🃏
+colorFrom: red
+colorTo: yellow
+sdk: gradio
+sdk_version: 5.20.0
+app_file: app.py
+pinned: false
+---
+
+# Mock JSON Generator
+
+Generate test data for a JSON schema.
+
+1. User inputs JSON schema
+2. Use Zero-Shot Classification model to map schema properties to faker functions
+3. Call faker functions and return mock JSON
+
+## Why
+
+Although mock data can be generated by chatting with any popular large language model (LLM), generating massive amounts of mock data for performance testing or load testing would be an inefficient use of an LLM. With this approach, once the classification model has mapped the schema properties, mock data can be generated instantly, for free, with no significant CPU load.
+
+Check the `case_study` directory to see why I decided to use Zero-Shot Classification for this solution.
+
+## Tradeoffs
+
+Using a Zero-Shot Classification model rather than a full LLM introduces a flexibility vs. scalability tradeoff. A full LLM would provide greater flexibility in understanding complex schemas and generating more nuanced mock data. However, this approach offers superior scalability - once the classification model has mapped schema properties to faker functions, you can generate virtually unlimited mock data instantly without the computational overhead or costs associated with repeated LLM calls.
+
+## Optimizations
+
+A custom-trained model would be ideal for a specific task like this.

app.py

@@ -0,0 +1,127 @@
+import json
+import gradio as gr
+from utils.classify import get_functions_for_descriptions
+from utils.extract import extract_descriptions
+from utils.fake import generate_mock_data
+from utils.validate import validate_schema_structure
+from utils.serialize_json import serialize_to_json
+
+# Store globally so we don't have to recalculate if
+# the schema has not changed
+function_mappings = {}
+
+
+def process_schema(schema):
+    """Process the schema and return either mock data or an error message"""
+    global function_mappings
+
+    is_valid, result = validate_schema_structure(schema)
+
+    if not is_valid:
+        return (
+            None,
+            result,  # If invalid this will contain the error message
+        )
+
+    if not function_mappings:
+        descriptions = extract_descriptions(result)
+        function_mappings = get_functions_for_descriptions(descriptions)
+
+    mock_data = generate_mock_data(result, function_mappings)
+
+    mock_json = serialize_to_json(mock_data, pretty=True)
+
+    return mock_json, None
+
+
+def clear_function_mappings():
+    global function_mappings
+    function_mappings = {}
+
+
+# Create a default schema example
+default_schema = json.dumps(
+    {
+        "type": "object",
+        "description": "A person object",
+        "properties": {
+            "first_name": {"type": "string", "description": "The person's first name"},
+            "last_name": {"type": "string", "description": "The person's last name"},
+            "age": {"type": "integer", "minimum": 18, "maximum": 100},
+            "email": {"type": "string"},
+            "is_active": {"type": "boolean"},
+            "address": {
+                "type": "object",
+                "properties": {
+                    "street": {"type": "string"},
+                    "city": {"type": "string"},
+                    "zip": {"type": "string"},
+                },
+            },
+        },
+        "required": ["first_name", "last_name", "age"],
+    },
+    indent=2,
+)
+
+# Create the Gradio interface
+with gr.Blocks(theme=gr.themes.Base()) as app:
+    gr.Markdown("# JSON Schema Mock Data Generator")
+    gr.Markdown(
+        "Enter a valid JSON schema and generate mock data that conforms to the schema."
+    )
+
+    with gr.Row():
+        with gr.Column():
+            schema_input = gr.Textbox(
+                label="JSON Schema",
+                value=default_schema,
+                lines=15,
+                placeholder="Enter your JSON schema here...",
+            )
+
+            generate_btn = gr.Button("Generate Mock Data", variant="primary")
+
+        with gr.Column():
+            mock_output = gr.Textbox(
+                label="Generated Mock JSON", lines=15, interactive=False
+            )
+
+            error_output = gr.Textbox(label="Errors", visible=False, interactive=False)
+
+    def update_output(schema_str):
+        mock_data, error = process_schema(schema_str)
+
+        if error:
+            return {
+                mock_output: None,
+                error_output: error,
+                error_output: gr.update(visible=True),
+            }
+        else:
+            return {
+                mock_output: mock_data,
+                error_output: None,
+                error_output: gr.update(visible=False),
+            }
+
+    schema_input.change(fn=clear_function_mappings, inputs=[], outputs=[])
+
+    generate_btn.click(
+        fn=update_output,
+        inputs=[schema_input],
+        outputs=[mock_output, error_output, error_output],
+    )
+
+    gr.Markdown(
+        """
+    ## Notes
+    - Zero Shot Classification will run the first time a mock is generated for a schema. Subsequent generations will be instant.
+    - The schema must be valid JSON and comply with JSON Schema Draft 7
+    - Required keywords: `type` and `properties` (for object types)
+    - Currently Supported types: string, integer, boolean, array, and object
+    """
+    )
+
+if __name__ == "__main__":
+    app.launch()

case_study/TableQuestionAnswering.ipynb

@@ -0,0 +1,174 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "50fa1916-c45f-4e79-aacd-8fd2956292b9",
+   "metadata": {},
+   "source": [
+    "# Table Question Answering Pipeline\n",
+    "\n",
+    "Experimenting with a TableQuestionAnsweringPipeline.\n",
+    "\n",
+    "This is not really a good use of the model but it kind of works."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "c4ca4235-042e-4b30-9a95-73f85c5f64ea",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import pipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "87419b8e-c1bb-42e4-b15a-e7d8c2b7367a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "faker_mappings = {\n",
+    "    # Personal information\n",
+    "      \"name\": [\n",
+    "            \"full name\", \"name\", \"name of user\", \"person name\", \"name of person\", \n",
+    "            \"complete name\", \"user name\", \"customer name\", \"client name\"\n",
+    "        ],\n",
+    "        \"first_name\": [\n",
+    "            \"first name\", \"user's first name\", \"first name of a person\", \"person first name\", \n",
+    "            \"given name\", \"forename\", \"christian name\", '', ''\n",
+    "        ],\n",
+    "        \"last_name\": [\n",
+    "            \"last name\", \"surname\", \"family name\", \"user's last name\", \n",
+    "            \"last name of user\", \"person's surname\", '', '', ''\n",
+    "        ],\n",
+    "        \"password\": [\n",
+    "            \"user password\", \"person password\", \"member password\", \"secret password\", \n",
+    "            \"confidential password\", \"example password\", '', '', '' # all arrays have to be equal. \n",
+    "        ]\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "1d2919a4-2312-4e8c-a1d3-66c7042b2400",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Device set to use mps:0\n"
+     ]
+    }
+   ],
+   "source": [
+    "pipe = pipeline(model=\"google/tapas-base-finetuned-wtq\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "0a7ca742-273c-4530-a591-3012fe31cc09",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/anaconda3/envs/llms/lib/python3.11/site-packages/transformers/models/tapas/tokenization_tapas.py:2699: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
+      "  text = normalize_for_match(row[col_index].text)\n",
+      "/opt/anaconda3/envs/llms/lib/python3.11/site-packages/transformers/models/tapas/tokenization_tapas.py:1493: FutureWarning: Series.__getitem__ treating keys as positions is deprecated. In a future version, integer keys will always be treated as labels (consistent with DataFrame behavior). To access a value by position, use `ser.iloc[pos]`\n",
+      "  cell = row[col_index]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[{'answer': 'first name', 'coordinates': [(0, 1)], 'cells': ['first name'], 'aggregator': 'NONE'}, {'answer': 'full name', 'coordinates': [(0, 0)], 'cells': ['full name'], 'aggregator': 'NONE'}, {'answer': 'secret password, confidential password', 'coordinates': [(3, 3), (4, 3)], 'cells': ['secret password', 'confidential password'], 'aggregator': 'NONE'}, {'answer': 'full name', 'coordinates': [(0, 0)], 'cells': ['full name'], 'aggregator': 'NONE'}]\n"
+     ]
+    }
+   ],
+   "source": [
+    "result = pipe(query=['user first name', 'full name', \"secret password\", \"unique id\"], table=faker_mappings)\n",
+    "print(result)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "803584e5-e3e1-48df-b974-712f7c5e81f5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# A function to lookup the faker function names based on coordinates\n",
+    "def lookup_keys_from_results(result, faker_mappings):\n",
+    "    reverse_mapping = {}\n",
+    "    for key, values in faker_mappings.items():\n",
+    "        for value in values:\n",
+    "            if value:\n",
+    "                reverse_mapping[value.lower()] = key\n",
+    "    \n",
+    "    coordinate_to_key = {}\n",
+    "    for item in result:\n",
+    "        answer = item['answer'].lower()\n",
+    "        \n",
+    "        if ',' in answer:\n",
+    "            answers = [a.strip() for a in answer.split(',')]\n",
+    "            for a in answers:\n",
+    "                if a in reverse_mapping:\n",
+    "                    for coord in item['coordinates']:\n",
+    "                        coordinate_to_key[coord] = reverse_mapping[a]\n",
+    "                    break\n",
+    "        else:\n",
+    "            if answer in reverse_mapping:\n",
+    "                for coord in item['coordinates']:\n",
+    "                    coordinate_to_key[coord] = reverse_mapping[answer]\n",
+    "    \n",
+    "    return coordinate_to_key"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "2025a892-e3e5-4efe-8822-2cbbd13f903c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{(0, 1): 'first_name', (0, 0): 'name', (3, 3): 'password', (4, 3): 'password'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(lookup_keys_from_results(result, faker_mappings))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

case_study/TextGeneration.ipynb

@@ -0,0 +1,199 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "b4614723-e45f-426e-a545-d0ed9685557d",
+   "metadata": {},
+   "source": [
+    "# Text Generation\n",
+    "\n",
+    "Experimenting with a Text Generation Pipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "b249c274-f0ab-43cb-9c44-b4de64045cac",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import pipeline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "09955617-16bb-4d6e-8546-91549c3fe296",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "2bbb30f80b274ff0b8c50a075f0cdd73",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some parameters are on the meta device because they were offloaded to the disk.\n",
+      "Device set to use mps\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Use a pre-quantized model from the Hub\n",
+    "pipe = pipeline(\n",
+    "    \"text-generation\",\n",
+    "    model=\"meta-llama/Llama-3.2-3B\",\n",
+    "    device_map=\"auto\",\n",
+    "    do_sample=True\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "4e7f1608-b823-4cee-ad31-c9a9c5feae4d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "first_name\n",
+      "email\n",
+      "phone\n"
+     ]
+    }
+   ],
+   "source": [
+    "def get_faker_mapping(properties):\n",
+    "    # Create a structured prompt\n",
+    "    prompt = \"\"\"\n",
+    "You are a data helper.\n",
+    "\n",
+    "Function Names:\n",
+    "name\n",
+    "first_name\n",
+    "last_name\n",
+    "email\n",
+    "phone\n",
+    "address\n",
+    "city\n",
+    "state\n",
+    "zip_code\n",
+    "country\n",
+    "company\n",
+    "job_title\n",
+    "credit_card_number\n",
+    "date_of_birth\n",
+    "text\n",
+    "random_number\n",
+    "\n",
+    "EXAMPLE 1:\n",
+    "Properties:\n",
+    "fullName: Full name of person\n",
+    "businessEmail: Business email address\n",
+    "phoneNum: US phone number\n",
+    "\n",
+    "Response:\n",
+    "fullName: name\n",
+    "businessEmail: email\n",
+    "phoneNum: phone\n",
+    "\n",
+    "EXAMPLE 2:\n",
+    "Properties:\n",
+    "name: name of person\n",
+    "ccn: Credit card number\n",
+    "personalPhone: US phone number\n",
+    "\n",
+    "Response:\n",
+    "name: name\n",
+    "ccn: credit_card_number\n",
+    "personalPhone: phone\n",
+    "\n",
+    "EXAMPLE 3:\n",
+    "Properties:\n",
+    "firstName: first name of person\n",
+    "city: City of residence\n",
+    "personalPhone: US phone number\n",
+    "\n",
+    "Response:\n",
+    "firstName: first_name\n",
+    "city: city\n",
+    "personalPhone: phone\n",
+    "\n",
+    "Complete EXAMPLE 4:\n",
+    "Properties:\n",
+    "\"\"\"\n",
+    "    # Add properties to prompt\n",
+    "    for prop, desc in properties:\n",
+    "        prompt += f\"{prop}: {desc}\\n\"\n",
+    "    \n",
+    "    prompt += \"\\nResponse:\"\n",
+    "    \n",
+    "    # Generate response\n",
+    "    response = pipe(\n",
+    "        prompt,\n",
+    "        max_new_tokens=50\n",
+    "    )\n",
+    "    \n",
+    "    result = response[0]['generated_text']\n",
+    "    \n",
+    "    # Logic to convert to a dict\n",
+    "    response_text = result.split(\"Complete EXAMPLE 4:\")[1].split(\"Complete EXAMPLE 5:\")[0].split(\"Response:\")[1].strip()\n",
+    "    \n",
+    "    return {line.split(\":\", 1)[0].strip(): line.split(\":\", 1)[1].strip() for line in response_text.splitlines() if \":\" in line}\n",
+    "    \n",
+    "# Example use\n",
+    "properties = [\n",
+    "    (\"name\", \"users first name\"),\n",
+    "    (\"email\", \"Business email address\"),\n",
+    "    (\"phone\", \"US phone number\")\n",
+    "]\n",
+    "\n",
+    "result = get_faker_mapping(properties)\n",
+    "\n",
+    "for key, desc in properties:\n",
+    "    print(result.get(key))\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

case_study/ZeroShotClassification.ipynb

@@ -0,0 +1,173 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "42aaa537-ab3b-46d2-a12b-d0355669bf26",
+   "metadata": {},
+   "source": [
+    "# ZeroShotClassification\n",
+    "\n",
+    "Experimenting with a ZeroShotClassification Pipeline.\n",
+    "\n",
+    "This works very good. Even \"login id\" was tagged as \"username\" at 20% confidence. Based on testing we will not consider below 18%."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "c8e72fc9-198e-43c6-8045-aaf811145e1c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import pipeline\n",
+    "from faker import Faker"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "620682d6-fd51-47b7-b99b-a32434c28044",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fake = Faker()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "0c5d29b2-2d9f-4009-9f55-690af4be7ea7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create a dictionary of Faker functions with descriptive labels\n",
+    "faker_functions = {\n",
+    "    \"person name\": fake.name,\n",
+    "    \"first name\": fake.first_name,\n",
+    "    \"last name\": fake.last_name,\n",
+    "    \"email address\": fake.email,\n",
+    "    \"phone number\": fake.phone_number,\n",
+    "    \"street address\": fake.street_address,\n",
+    "    \"city name\": fake.city,\n",
+    "    \"state name\": fake.state,\n",
+    "    \"country name\": fake.country,\n",
+    "    \"zip code\": fake.zipcode,\n",
+    "    \"job title\": fake.job,\n",
+    "    \"company name\": fake.company,\n",
+    "    \"credit card number\": fake.credit_card_number,\n",
+    "    \"date of birth\": fake.date_of_birth,\n",
+    "    \"username\": fake.user_name,\n",
+    "    \"website url\": fake.url,\n",
+    "    \"paragraph text\": fake.paragraph,\n",
+    "    \"sentence text\": fake.sentence\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "998e0fa8-0ed5-4b39-8f5e-732628ace900",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Device set to use mps:0\n"
+     ]
+    }
+   ],
+   "source": [
+    "pipe = pipeline(model=\"facebook/bart-large-mnli\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "1a1d7010-1291-4e31-b525-83327b5e7c01",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result = pipe(\n",
+    "    [\"The first name of a user\", \"login id\", \"full name of member\"],\n",
+    "    candidate_labels=list(faker_functions.keys())\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "d8cd7a50-aa79-48e4-aba9-7e78a4e64074",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_highest_score_functions(result, faker_functions, threshold=0.18):\n",
+    "    sequence_to_function = {}\n",
+    "    \n",
+    "    for item in result:\n",
+    "        sequence = item['sequence']\n",
+    "        label = item['labels'][0]\n",
+    "        score = item['scores'][0]\n",
+    "        \n",
+    "        if (score >= threshold):\n",
+    "            sequence_to_function[sequence] = faker_functions.get(label)\n",
+    "        else:\n",
+    "            sequence_to_function[sequence] = None\n",
+    "    \n",
+    "    return sequence_to_function"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "5c1989e2-99b8-4de3-b7f9-45c625405eb9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'The first name of a user': <bound method Provider.first_name of <faker.providers.person.en_US.Provider object at 0x34cecc050>>,\n",
+       " 'login id': <bound method Provider.user_name of <faker.providers.internet.en_US.Provider object at 0x34ceb2cd0>>,\n",
+       " 'full name of member': <bound method Provider.name of <faker.providers.person.en_US.Provider object at 0x34cecc050>>}"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "get_highest_score_functions(result, faker_functions, threshold=0.18)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "799de95c-3d08-4da9-8a84-c3bc81ab5928",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

requirements.txt

@@ -0,0 +1,7 @@
+python-dotenv
+huggingface-hub
+transformers
+faker
+torch
+jsonschema
+gradio

utils/classify.py

@@ -0,0 +1,80 @@
+"""
+Utility for generating mock data functions based on text descriptions.
+
+This utility uses a zero-shot classification model to match text descriptions
+to appropriate Faker functions for generating synthetic data.
+"""
+
+from transformers import pipeline
+from faker import Faker
+
+fake = Faker()
+
+# Create a dictionary of Faker functions with descriptive labels
+faker_functions = {
+    "person name": fake.name,
+    "first name": fake.first_name,
+    "last name": fake.last_name,
+    "email address": fake.email,
+    "phone number": fake.phone_number,
+    "street address": fake.street_address,
+    "city name": fake.city,
+    "state name": fake.state,
+    "country name": fake.country,
+    "zip code": fake.zipcode,
+    "job title": fake.job,
+    "company name": fake.company,
+    "credit card number": fake.credit_card_number,
+    "date of birth": fake.date_of_birth,
+    "username": fake.user_name,
+    "website url": fake.url,
+    "paragraph text": fake.paragraph,
+    "sentence text": fake.sentence,
+}
+
+
+def get_highest_score_functions(result, faker_functions, threshold=0.18):
+    """
+    Process model results and map sequences to appropriate Faker functions.
+
+    Args:
+        result: The classification results from the model
+        faker_functions: Dictionary mapping labels to Faker functions
+        threshold: Minimum confidence score to assign a function (default: 0.18)
+
+    Returns:
+        dict: Mapping of input sequences to corresponding Faker functions
+    """
+    sequence_to_function = {}
+    for item in result:
+        sequence = item["sequence"]
+        label = item["labels"][0]
+        score = item["scores"][0]
+        if score >= threshold:
+            sequence_to_function[sequence] = faker_functions.get(label)
+        else:
+            sequence_to_function[sequence] = None
+    return sequence_to_function
+
+
+def get_functions_for_descriptions(descriptions):
+    """
+    Get mock data functions based on descriptions or property names.
+
+    Uses zero-shot classification to match text descriptions to appropriate
+    Faker functions for generating synthetic data.
+
+    Args:
+        descriptions: Array of descriptions or property names to classify
+
+    Returns:
+        dict: Mapping of descriptions to corresponding mock data functions
+    """
+    # Create pipeline with Facebook's BART model for zero-shot classification
+    pipe = pipeline(model="facebook/bart-large-mnli")
+
+    # Call pipeline with descriptions and available Faker function labels
+    result = pipe(descriptions, candidate_labels=list(faker_functions.keys()))
+
+    # Process results using helper function with default confidence threshold
+    return get_highest_score_functions(result, faker_functions, threshold=0.18)

utils/extract.py

@@ -0,0 +1,89 @@
+def extract_descriptions(schema, path=""):
+    """
+    Recursively extract all descriptions from a JSON schema.
+    If a property doesn't have a description, use its property name.
+
+    Args:
+        schema (dict): The JSON schema to extract descriptions from
+        path (str): Current path in the schema (for nested properties)
+
+    Returns:
+        list: All descriptions found in the schema
+    """
+    descriptions = []
+
+    # Handle schema description or use path as fallback
+    # Skip if type is object
+    if schema.get("type") != "object":
+        if "description" in schema:
+            descriptions.append(schema["description"])
+        elif path:  # Only add path if it's not empty (root element)
+            descriptions.append(path.split(".")[-1])  # Use the last part of the path
+
+    # Handle properties
+    if "properties" in schema:
+        for prop_name, prop_schema in schema["properties"].items():
+            prop_path = f"{path}.{prop_name}" if path else prop_name
+            descriptions.extend(extract_descriptions(prop_schema, prop_path))
+
+    # Handle array items
+    if "items" in schema and isinstance(schema["items"], dict):
+        item_path = f"{path}[]" if path else "items"
+        descriptions.extend(extract_descriptions(schema["items"], item_path))
+
+    # Handle oneOf, anyOf, allOf
+    for key in ["oneOf", "anyOf", "allOf"]:
+        if key in schema and isinstance(schema[key], list):
+            for i, sub_schema in enumerate(schema[key]):
+                sub_path = f"{path}.{key}[{i}]" if path else f"{key}[{i}]"
+                descriptions.extend(extract_descriptions(sub_schema, sub_path))
+
+    # Handle additional properties
+    if "additionalProperties" in schema and isinstance(
+        schema["additionalProperties"], dict
+    ):
+        add_path = f"{path}.additionalProperties" if path else "additionalProperties"
+        descriptions.extend(
+            extract_descriptions(schema["additionalProperties"], add_path)
+        )
+
+    return descriptions
+
+
+# Example usage:
+if __name__ == "__main__":
+    sample_schema = {
+        "type": "object",
+        "description": "A person object",
+        "properties": {
+            "name": {"type": "string", "description": "The person's full name"},
+            "age": {
+                "type": "integer",
+                # No description for age, will use property name
+            },
+            "address": {
+                "type": "object",
+                "description": "The person's address",
+                "properties": {
+                    "street": {
+                        "type": "string",
+                        "description": "Street name and number",
+                    },
+                    "city": {
+                        "type": "string"
+                        # No description for city, will use property name
+                    },
+                },
+            },
+            "hobbies": {
+                "type": "array",
+                "description": "List of hobbies",
+                "items": {"type": "string", "description": "A hobby name"},
+            },
+        },
+    }
+
+    descriptions = extract_descriptions(sample_schema)
+    print("Extracted descriptions:")
+    for desc in descriptions:
+        print(f"- {desc}")

utils/fake.py

@@ -0,0 +1,95 @@
+import random
+
+def generate_mock_data(schema, faker_mapping):
+    """
+    Generate mock data based on a JSON schema.
+    For integer and boolean, use custom logic to generate fake data
+    For remaining types, call the corresponding faker function if available.
+
+    Args:
+        schema (dict): The JSON schema
+        faker_mapping (dict): Mapping of descriptions/property names to faker functions
+
+    Returns:
+        dict: Generated mock data
+    """
+    result = {}
+
+    if "properties" in schema:
+        for prop_name, prop_schema in schema["properties"].items():
+            if prop_schema.get("type") == "object":
+                # Recursively process objects
+                result[prop_name] = generate_mock_data(prop_schema, faker_mapping)
+            elif prop_schema.get("type") == "integer":
+                # Check if min/max properties exist for bounds
+                minimum = prop_schema.get("minimum")
+                maximum = prop_schema.get("maximum")
+
+                if minimum is not None and maximum is not None:
+                    result[prop_name] = random.randint(minimum, maximum)
+                elif minimum is not None:
+                    result[prop_name] = random.randint(
+                        minimum, minimum + 100
+                    )  # Arbitrary upper bound
+                elif maximum is not None:
+                    result[prop_name] = random.randint(
+                        0, maximum
+                    )  # Assume 0 as lower bound
+                else:
+                    result[prop_name] = random.randint(0, 100)  # Default range
+            elif prop_schema.get("type") == "boolean":
+                result[prop_name] = random.choice(
+                    [True, False]
+                )  # Randomly pick True or False
+            else:
+                # For non-objects, try to find a faker function
+                description = prop_schema.get("description", "")
+
+                # Look for faker function by description or property name
+                faker_func = None
+                if description in faker_mapping:
+                    faker_func = faker_mapping[description]
+                elif prop_name in faker_mapping:
+                    faker_func = faker_mapping[prop_name]
+
+                # Call faker function if found, otherwise set to None
+                result[prop_name] = faker_func() if faker_func else None
+
+    return result
+
+
+# Example usage:
+if __name__ == "__main__":
+    from faker import Faker
+
+    fake = Faker()
+
+    sample_schema = {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string", "description": "The person's full name"},
+            "age": {"type": "integer"},
+            "address": {
+                "type": "object",
+                "properties": {
+                    "street": {
+                        "type": "string",
+                        "description": "Street name and number",
+                    },
+                    "city": {"type": "string"},
+                },
+            },
+        },
+    }
+
+    # Create faker function mapping
+    faker_mapping = {
+        "The person's full name": fake.name,
+        "age": fake.random_int,
+        "Street name and number": fake.street_address,
+        "city": fake.city,
+    }
+
+    # Generate mock data
+    mock_data = generate_mock_data(sample_schema, faker_mapping)
+    print(mock_data)

utils/serialize_json.py

@@ -0,0 +1,65 @@
+import json
+import datetime
+from decimal import Decimal
+from uuid import UUID
+
+
+class CustomJSONEncoder(json.JSONEncoder):
+    """
+    Custom JSON encoder that handles various Python types that aren't JSON serializable by default:
+    - datetime.datetime, datetime.date, datetime.time
+    - Decimal
+    - UUID
+    - Sets
+    - Any object with a to_json method
+    - Custom objects with __dict__ attribute
+    """
+
+    def default(self, obj):
+        # Handle datetime objects
+        if isinstance(obj, (datetime.datetime, datetime.date)):
+            return obj.isoformat()
+
+        # Handle time objects
+        if isinstance(obj, datetime.time):
+            return obj.isoformat()
+
+        # Handle Decimal objects
+        if isinstance(obj, Decimal):
+            return float(obj)
+
+        # Handle UUID objects
+        if isinstance(obj, UUID):
+            return str(obj)
+
+        # Handle set objects
+        if isinstance(obj, set):
+            return list(obj)
+
+        # Handle objects that have implemented a to_json method
+        if hasattr(obj, "to_json"):
+            return obj.to_json()
+
+        # Try to convert the object to a dict
+        try:
+            return obj.__dict__
+        except AttributeError:
+            pass
+
+        # Let the base class handle it or raise TypeError
+        return super().default(obj)
+
+
+def serialize_to_json(data, pretty=False):
+    """
+    Serialize Python data to JSON string, handling various Python types.
+
+    Args:
+        data: The Python object to serialize
+        pretty (bool): Whether to format the JSON with indentation for readability
+
+    Returns:
+        str: JSON string representation of the data
+    """
+    indent = 2 if pretty else None
+    return json.dumps(data, cls=CustomJSONEncoder, indent=indent)

utils/validate.py

@@ -0,0 +1,34 @@
+import json
+from jsonschema import validate, exceptions
+
+meta_schema = {
+    "type": "object",
+    "required": ["type", "properties"],
+    "properties": {"type": {"type": "string"}, "properties": {"type": "object"}},
+}
+
+
+def validate_schema_structure(schema_str):
+    """
+    Parse and Validate a JSON Schema object
+
+    Args:
+        schema_str: A valid JSON schema object seralized to a string
+
+        Returns:
+        tuple: (bool, object) where:
+            - First element is a boolean indicating success (True) or failure (False)
+            - Second element is either the validated schema object (on success)
+              or an error message string (on failure)
+    """
+
+    try:
+        schema = json.loads(schema_str)
+        validate(instance=schema, schema=meta_schema)
+        return True, schema
+    except exceptions.ValidationError as e:
+        return False, f"Schema structure validation error: {e}"
+    except json.JSONDecodeError as e:
+        return False, f"Invalid JSON syntax: {str(e)}"
+    except Exception as e:
+        return False, f"Unexpected error: {str(e)}"