jalr/tabbyAPI-ollama
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Requirements: Split for configurations

Browse source 
Add self-contained requirements for cuda 11.8 and ROCm

Signed-off-by: kingbri <bdashore3@proton.me>
This commit is contained in:
kingbri 2023-12-06 00:00:30 -05:00
parent 4c0e686e7d
commit b83e1b704e
5 changed files with 88 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

73
TabbyAPI_Colab_Example.ipynb
View file

@ -1,43 +1,27 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"gpuType": "T4"
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
},
"accelerator": "GPU"
},
"cells": [
{
"cell_type": "markdown",
"source": [
"# **TabbyAPI Colab**"
],
"metadata": {
"id": "NcgQp3r7BS-q"
}
},
"source": [
"# **TabbyAPI Colab**"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "zU0omxLLm8AN"
},
"outputs": [],
"source": [
"#@title # Keep this widget playing to prevent Colab from disconnecting you { display-mode: \"form\" }\n",
"#@markdown Press play on the audio player that will appear below:\n",
"%%html\n",
"<audio src=\"https://oobabooga.github.io/silence.m4a\" controls>"
],
"metadata": {
"id": "zU0omxLLm8AN"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
@ -63,8 +47,9 @@
"\n",
"!git clone https://github.com/theroyallab/tabbyAPI\n",
"%cd tabbyAPI\n",
"!pip install https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu118-cp310-cp310-linux_x86_64.whl -q\n",
"!pip install -r requirements-colab.txt -q\n",
"\n",
"# Install cuda 11.8 requirements\n",
"!pip install -r requirements-cu118.txt -q\n",
"!pip install huggingface-hub -q\n",
"\n",
"# Download cloudflared tunnel\n",
@ -83,6 +68,11 @@
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "aD7Rz8ZCeShD"
},
"outputs": [],
"source": [
"# @title # Configure and launch API { display-mode: \"form\" }\n",
"# @markdown ---\n",
@ -179,12 +169,23 @@
"%cd /content/tabbyAPI/\n",
"\n",
"!python main.py & ./cloudflared tunnel --url localhost:5000"
],
"metadata": {
"id": "aD7Rz8ZCeShD"
},
"execution_count": null,
"outputs": []
]
}
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"gpuType": "T4",
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

13
requirements-amd.txt Normal file
View file

@ -0,0 +1,13 @@
# Torch
torch --index-url https://download.pytorch.org/whl/rocm5.6
# Exllamav2
https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+rocm5.6-cp311-cp311-linux_x86_64.whl
https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+rocm5.6-cp310-cp310-linux_x86_64.whl
# Pip dependencies
fastapi
pydantic < 2,>= 1
PyYAML
progress
uvicorn

10
requirements-colab.txt
View file

@ -1,10 +0,0 @@
fastapi
pydantic < 2,>= 1
PyYAML
progress
uvicorn
# Wheels
# Linux FA2 from https://github.com/Dao-AILab/flash-attention/releases
https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.6/flash_attn-2.3.6+cu118torch2.1cxx11abiFALSE-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10"

22
requirements-cu118.txt Normal file
View file

@ -0,0 +1,22 @@
# Torch
torch --index-url https://download.pytorch.org/whl/cu118
# Exllamav2
# Windows
https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu118-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu118-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10"
# Linux
https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu118-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu118-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10"
# Pip dependencies
fastapi
pydantic < 2,>= 1
PyYAML
progress
uvicorn
# Linux FA2 from https://github.com/Dao-AILab/flash-attention/releases
https://github.com/Dao-AILab/flash-attention/releases/download/v2.3.6/flash_attn-2.3.6+cu118torch2.1cxx11abiFALSE-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10"

18
requirements.txt
View file

@ -1,10 +1,24 @@
fastapi
# Torch
torch --index-url https://download.pytorch.org/whl/cu121
# Exllamav2
# Windows
https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu121-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"
https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu121-cp310-cp310-win_amd64.whl; platform_system == "Windows" and python_version == "3.10"
# Linux
https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu121-cp311-cp311-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.11"
https://github.com/turboderp/exllamav2/releases/download/v0.0.10/exllamav2-0.0.10+cu121-cp310-cp310-linux_x86_64.whl; platform_system == "Linux" and platform_machine == "x86_64" and python_version == "3.10"
# Pip dependencies
fastapi
pydantic < 2,>= 1
PyYAML
progress
uvicorn
# Wheels
# Flash attention v2
# Windows FA2 from https://github.com/jllllll/flash-attention/releases
https://github.com/jllllll/flash-attention/releases/download/v2.3.6/flash_attn-2.3.6+cu121torch2.1cxx11abiFALSE-cp311-cp311-win_amd64.whl; platform_system == "Windows" and python_version == "3.11"