From b83e1b704ecca6388709a65dcdbea709dd4e1320 Mon Sep 17 00:00:00 2001
From: kingbri <bdashore3@proton.me>
Date: Wed, 6 Dec 2023 00:00:30 -0500
Subject: [PATCH] Requirements: Split for configurations

Add self-contained requirements for cuda 11.8 and ROCm

Signed-off-by: kingbri <bdashore3@proton.me>
---
 TabbyAPI_Colab_Example.ipynb | 73 ++++++++++++++++++------------------
 requirements-amd.txt         | 13 +++++++
 requirements-colab.txt       | 10 -----
 requirements-cu118.txt       | 22 +++++++++++
 requirements.txt             | 18 ++++++++-
 5 files changed, 88 insertions(+), 48 deletions(-)
 create mode 100644 requirements-amd.txt
 delete mode 100644 requirements-colab.txt
 create mode 100644 requirements-cu118.txt

diff --git a/TabbyAPI_Colab_Example.ipynb b/TabbyAPI_Colab_Example.ipynb
index c36fab5..8a5b584 100644
--- a/TabbyAPI_Colab_Example.ipynb
+++ b/TabbyAPI_Colab_Example.ipynb
@@ -1,43 +1,27 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": [],
-      "gpuType": "T4"
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    },
-    "accelerator": "GPU"
-  },
   "cells": [
     {
       "cell_type": "markdown",
-      "source": [
-        "# **TabbyAPI Colab**"
-      ],
       "metadata": {
         "id": "NcgQp3r7BS-q"
-      }
+      },
+      "source": [
+        "# **TabbyAPI Colab**"
+      ]
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "zU0omxLLm8AN"
+      },
+      "outputs": [],
       "source": [
         "#@title # Keep this widget playing to prevent Colab from disconnecting you { display-mode: \"form\" }\n",
         "#@markdown Press play on the audio player that will appear below:\n",
         "%%html\n",
         "<audio src=\"https://oobabooga.github.io/silence.m4a\" controls>"
-      ],
-      "metadata": {
-        "id": "zU0omxLLm8AN"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     },
     {
       "cell_type": "code",
@@ -63,8 +47,9 @@
         "\n",
         "!git clone https://github.com/theroyallab/tabbyAPI\n",
         "%cd tabbyAPI\n",
-        "!pip install https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu118-cp310-cp310-linux_x86_64.whl -q\n",
-        "!pip install -r requirements-colab.txt -q\n",
+        "\n",
+        "# Install cuda 11.8 requirements\n",
+        "!pip install -r requirements-cu118.txt -q\n",
         "!pip install huggingface-hub -q\n",
         "\n",
         "# Download cloudflared tunnel\n",
@@ -83,6 +68,11 @@
     },
     {
       "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "aD7Rz8ZCeShD"
+      },
+      "outputs": [],
       "source": [
         "# @title # Configure and launch API { display-mode: \"form\" }\n",
         "# @markdown ---\n",
@@ -179,12 +169,23 @@
         "%cd /content/tabbyAPI/\n",
         "\n",
         "!python main.py & ./cloudflared tunnel --url localhost:5000"
-      ],
-      "metadata": {
-        "id": "aD7Rz8ZCeShD"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
     }
-  ]
-}
\ No newline at end of file
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "gpuType": "T4",
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
diff --git a/requirements-amd.txt b/requirements-amd.txt
new file mode 100644
index 0000000..42be584
--- /dev/null
+++ b/requirements-amd.txt
@@ -0,0 +1,13 @@
+# Torch
+torch --index-url https://download.pytorch.org/whl/rocm5.6
+
+# Exllamav2
+https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+rocm5.6-cp311-cp311-linux_x86_64.whl
+https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+rocm5.6-cp310-cp310-linux_x86_64.whl
+
+# Pip dependencies
+fastapi
+pydantic < 2,>= 1
+PyYAML
+progress
+uvicorn
diff --git a/requirements-colab.txt b/requirements-colab.txt
deleted file mode 100644
index e092c0a..0000000
--- a/requirements-colab.txt
+++ /dev/null
@@ -1,10 +0,0 @@
-fastapi
-pydantic < 2,>= 1
-PyYAML
-progress
-uvicorn
-
-# Wheels
-
-# Linux FA2 from https://github.com/Dao-AILab/flash-attention/releases
-https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.6/flash_attn-2.3.6+cu118torch2.1cxx11abiFALSE-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10"
diff --git a/requirements-cu118.txt b/requirements-cu118.txt
new file mode 100644
index 0000000..563bf04
--- /dev/null
+++ b/requirements-cu118.txt
@@ -0,0 +1,22 @@
+# Torch
+torch --index-url https://download.pytorch.org/whl/cu118
+
+# Exllamav2
+
+# Windows
+https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu118-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu118-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10"
+
+# Linux
+https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu118-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu118-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10"
+
+# Pip dependencies
+fastapi
+pydantic < 2,>= 1
+PyYAML
+progress
+uvicorn
+
+# Linux FA2 from https://github.com/Dao-AILab/flash-attention/releases
+https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.6/flash_attn-2.3.6+cu118torch2.1cxx11abiFALSE-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10"
diff --git a/requirements.txt b/requirements.txt
index c73e597..1a93c08 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,24 @@
-fastapi
+# Torch
+torch --index-url https://download.pytorch.org/whl/cu121
+
+# Exllamav2
+
+# Windows
+https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
+https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu121-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10"
+
+# Linux
+https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu121-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
+https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu121-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10"
+
+# Pip dependencies
+fastapi
 pydantic < 2,>= 1
 PyYAML
 progress
 uvicorn
 
-# Wheels
+# Flash attention v2
 
 # Windows FA2 from https://github.com/jllllll/flash-attention/releases
 https://github.com/jllllll/flash-attention/releases/download/v2.3.6/flash_attn-2.3.6+cu121torch2.1cxx11abiFALSE-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"