candle-vllm/Cargo.toml at master · mwaurac/candle-vllm · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
[package]
name = "candle-vllm"
version = "0.9.0"
edition = "2021"
default-run = "candle-vllm"

[lib]
name = "candle_vllm"
path = "src/lib.rs"

[[bin]]
name = "candle-vllm"
path = "src/main.rs"

[dependencies]
axum = { version = "0.8.9", features = ["tokio"] }
utoipa = { version = "4.2", features = ["axum_extras"] }
tower-http = { version = "0.6.6", features = ["cors"]}
#actix-web = "4.8.0"
anyhow = "1.0.75"
rand = "0.9.0"
rayon="1.10.0"
hyper = { version = "0.14", features = ["full"] }
candle-core = { git = "https://github.com/guoqingbao/candle.git", version = "0.8.3", rev = "394df6e" }
candle-nn = { git = "https://github.com/guoqingbao/candle.git", version = "0.8.3", rev = "394df6e" }
dyn-fmt = "0.4.0"
safetensors = "0.8"
serde = { version = "1.0.190", features = ["serde_derive"] }
tokenizers = "0.21.2"
uuid = { version = "1.5.0", features = ["v4"] }
hf-hub = "0.4.1"
serde_json = "1.0.108"
derive_more = "0.99.17"
accelerate-src = { version = "0.3.2", optional = true }
intel-mkl-src = { version = "0.8.1", features = ["mkl-static-lp64-iomp"], optional = true }
half = { version = "2.5.0", features = ["num-traits", "use-intrinsics", "rand_distr"] }
clap = { version = "4.4.7", features = ["derive"] }
futures = "0.3.29"
tokio = { version = "1.38.0", features = ["sync"] }
env_logger = "0.10.1"
tracing = "0.1.40"
range-checked = { git = "https://github.com/EricLBuehler/range-checked.git", version = "0.1.0" }
either = { version = "1.13.0", features = ["serde"] }
dirs = "5.0.1"
minijinja = { version = "2.10.2", features = ["builtins", "json"] }
minijinja-contrib = { version = "2.10.2", features = ["pycompat"] }
thiserror = "1.0.58"
attention-rs = { git = "https://github.com/guoqingbao/attention.rs.git", version="0.6.6", rev = "2e0db42" }
metal = { version = "0.27.0", features = ["mps"], optional = true }
lazy_static = {version = "1.4.0"}
interprocess = "2.2.2"
serde-big-array = "0.5.1"
bincode = { version = "1.3.1" }
ftail = "0.2"
tqdm = "0.8.0"
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
indicatif = "0.17.11"
parking_lot = "0.12"
akin = "0.4.0"
itertools = "0.13.0"
chrono = "0.4.41"
which = "8.0.0"
ahash = "0.8.11"
rustchatui = "0.3.2"
base64 = "0.22.1"
bytes = "1.5"
reqwest = { version = "0.12.24", features = ["blocking", "json", "rustls-tls"]}
regex = "1.12.2"
tool-parser = "1.3.1"
openai-protocol = "1.8.1"
crossbeam = "0.8.4"
colored = { version = "3.0.0" }
local-ip-address = "0.6.5"
image = { version = "0.25.6", default-features = false, features = ["bmp", "gif", "jpeg", "png", "tiff", "webp"] }
bytemuck = "1.23.2"
libc = "0.2.184"

[features]
accelerate = ["dep:accelerate-src", "candle-core/accelerate", "candle-nn/accelerate"]
cuda = ["candle-core/cuda", "candle-nn/cuda", "attention-rs/cuda", "graph"]
cutlass = ["cuda", "attention-rs/cutlass"]
flash = ["attention-rs/flash"]
metal = ["candle-core/metal", "candle-nn/metal", "dep:metal", "attention-rs/metal", "attention-rs/metal-flash"]
cudnn = ["candle-core/cudnn"]
flashattn = ["attention-rs/flashattn"]
flashinfer = ["attention-rs/flashinfer"]
mkl = ["dep:intel-mkl-src", "candle-core/mkl", "candle-nn/mkl"]
nccl = ["candle-core/nccl"]
graph = ["flash", "attention-rs/graph", "candle-core/graph"]

[target.'cfg(target_os = "linux")'.dependencies]
openssl-sys = { version = "0.9", features = ["vendored"] }

[package.metadata.deb]
maintainer = "guoqingbao"
copyright = "2024, guoqingbao"
extended-description = "candle-vllm — a high-performance LLM inference engine in Rust with CUDA/Metal acceleration"
depends = "$auto"
section = "utils"
priority = "optional"
assets = [
    ["target/release/candle-vllm", "usr/local/bin/candle-vllm", "755"],
    ["target/release/lib/*", "usr/local/lib/candle-vllm/", "644"],
]