{
  "version": "https://jsonfeed.org/version/1.1",
  "title": "Thoughts — Paymanshu Sharma",
  "home_page_url": "https://paymanshus.github.io/thoughts/",
  "feed_url": "https://paymanshus.github.io/feed.json",
  "description": "Essays on machine learning, agents, and engineering by Paymanshu Sharma.",
  "language": "en-us",
  "authors": [
    {
      "name": "Paymanshu Sharma",
      "url": "https://paymanshus.github.io/"
    }
  ],
  "items": [
    {
      "id": "https://paymanshus.github.io/thoughts/llm-optimization-memory-gpu-cpu/",
      "url": "https://paymanshus.github.io/thoughts/llm-optimization-memory-gpu-cpu/",
      "title": "Fitting Really Big Models into Not-So-Big GPUs: Precision and Quantization",
      "summary": "How much GPU memory a model actually needs and how to cut it down — fp32, fp16 and bf16, what quantization trades away, LLM.int8() and bitsandbytes in practice, and the KV cache that grows on top of the weights.",
      "date_published": "2024-12-14T00:00:00.000Z",
      "tags": [
        "llm optimization",
        "quantization",
        "precision",
        "kv cache",
        "vram",
        "inference"
      ],
      "image": "https://paymanshus.github.io/og/thoughts/llm-optimization-memory-gpu-cpu.png",
      "authors": [
        {
          "name": "Paymanshu Sharma",
          "url": "https://paymanshus.github.io/"
        }
      ]
    },
    {
      "id": "https://paymanshus.github.io/thoughts/on-device-llms-on-android/",
      "url": "https://paymanshus.github.io/thoughts/on-device-llms-on-android/",
      "title": "On-Device Large Language Models on Android",
      "summary": "Why running LLMs on phones is worth the trouble, and two working Android demos — a RAG document-querying app built on Phi-3-mini and ONNX Runtime, and a multi-LoRA adapter-switching app on Gemma 2B.",
      "date_published": "2024-12-14T00:00:00.000Z",
      "tags": [
        "on-device ai",
        "edge llm",
        "onnx runtime",
        "quantization",
        "android",
        "rag"
      ],
      "image": "https://paymanshus.github.io/og/thoughts/on-device-llms-on-android.png",
      "authors": [
        {
          "name": "Paymanshu Sharma",
          "url": "https://paymanshus.github.io/"
        }
      ]
    }
  ]
}