用 Colab 免費弄一個自己的 AI 程式碼助手

你的本機 Windows (OpenCode)
        ↕ 透過 HTTPS 傳輸
    ngrok 幫你穿透內網的公開網址
        ↕
  Colab 幫你頂著的 FastAPI 伺服器
        ↕
   正在 T4 GPU 裡瘋狂運算的神經網路 (Gemma)

!pip install fastapi uvicorn pyngrok nest_asyncio transformers accelerate bitsandbytes

from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
import torch

# 正確的二代小模型 ID
model_id = "google/gemma-2-2b-it"

# 4-bit 量化設定：不開這個硬體很容易炸掉
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.float16,
)

# 記得先去 Hugging Face 勾選同意 Gemma 的使用條款，並把你的 HF_TOKEN 填進來
HF_TOKEN = "你的_HUGGING_FACE_TOKEN"

tokenizer = AutoTokenizer.from_pretrained(model_id, token=HF_TOKEN)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map="auto",
    quantization_config=bnb_config,
    token=HF_TOKEN
)

from fastapi import FastAPI, Response
from pydantic import BaseModel
from typing import List, Optional, Any
import json, asyncio

app = FastAPI()

class Message(BaseModel):
    role: str
    content: Optional[str] = None

class CompletionRequest(BaseModel):
    model: str = "gemma-2-2b-it"
    messages: List[Message]
    stream: bool = False

def generate_response(messages: list) -> str:
    # 抓最後那條 user 說的話
    prompt = messages[-1]["content"]
    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
    with torch.no_grad():
        outputs = model.generate(**inputs, max_new_tokens=512)
    new_tokens = outputs[0][inputs.input_ids.shape[1]:]
    return tokenizer.decode(new_tokens, skip_special_tokens=True).strip()

@app.get("/v1/models")
def list_models():
    return {
        "object": "list",
        "data": [{"id": "gemma-2-2b-it", "object": "model", "owned_by": "google"}]
    }

@app.post("/v1/chat/completions")
async def chat_completions(request: CompletionRequest):
    messages = [m.dict(exclude_none=True) for m in request.messages]
    loop = asyncio.get_event_loop()
    reply = await loop.run_in_executor(None, generate_response, messages)

    message = {"role": "assistant", "content": reply}

    # 簡單模擬一下 Stream 串流輸出，騙過前端插件
    if request.stream:
        sse  = f"data: {json.dumps({'id':'chatcmpl-1','object':'chat.completion.chunk','model':request.model,'choices':[{'index':0,'delta':message,'finish_reason':None}]})}\n\n"
        sse += f"data: {json.dumps({'id':'chatcmpl-1','object':'chat.completion.chunk','model':request.model,'choices':[{'index':0,'delta':{},'finish_reason':'stop'}]})}\n\n"
        sse += "data: [DONE]\n\n"
        return Response(content=sse, media_type="text/event-stream")

    return {
        "id": "chatcmpl-1",
        "object": "chat.completion",
        "model": request.model,
        "choices": [{"index": 0, "message": message, "finish_reason": "stop"}]
    }

from pyngrok import ngrok
import uvicorn, threading, nest_asyncio

nest_asyncio.apply()

# 去 ngrok 官網註冊一個免費帳號就能拿到 Token
ngrok.set_auth_token("你的_NGROK_TOKEN")
public_url = ngrok.connect(8000)
print(f"🔗 你的專屬 API 網址：{public_url.public_url}")

def run_server():
    uvicorn.run(app, host="0.0.0.0", port=8000)

threading.Thread(target=run_server, daemon=True).start()

npm install -g opencode-ai

{
  "$schema": "https://opencode.ai/config.json",
  "provider": {
    "gemma": {
      "api": "openai",
      "options": {
        "baseURL": "https://你的ngrok網址/v1", // ⚠️ 後面一定要記得加 /v1，不然會迷路！
        "apiKey": "dummy" // 隨便亂填沒關係，因為我們沒設密碼
      },
      "models": {
        "gemma-2-2b-it": {
          "id": "gemma-2-2b-it",
          "limit": {
            "context": 8192,
            "output": 1024
          }
        }
      }
    }
  },
  "model": "gemma/gemma-2-2b-it"
}

opencode