Gemma 4 Edge Deployment: From Data Center to Pocket

1import litert_lm
2
3interpreter = litert_lm.Interpreter(
4    model_path="gemma-4-e2b-int8.tflite",
5    num_threads=4  # Use all Raspberry Pi 5 cores
6)
7
8interpreter.invoke(prompt="Hello, Gemma!")

1// Modern Android (API 34+) with AICore
2class GemmaInferenceService : Service() {
3    private lateinit var generativeModel: GenerativeModel
4    
5    override fun onCreate() {
6        super.onCreate()
7        
8        // System-defined configuration for edge inference
9        val config = InferenceConfiguration.Builder()
10            .setTemperature(0.7f)
11            .setMaxOutputTokens(1024)
12            .setTopK(40)
13            .build()
14            
15        generativeModel = GenerativeModel(
16            modelName = "gemma-4-e2b",
17            config = config,
18            context = applicationContext
19        )
20    }
21    
22    fun analyzeImage(bitmap: Bitmap, userQuery: String): Flow<String> {
23        return generativeModel.generateContentStream(
24            content {
25                image(bitmap)
26                text(userQuery)
27            }
28        ).map { it.text ?: "" }
29    }
30}

1import MLX
2import MLXLLM
3
4// Convert Gemma 4 E2B to Core ML or run via MLX
5let modelConfiguration = ModelConfiguration(
6    id: "google/gemma-4-e2b",
7    overrideTokenizer: "PreTrainedTokenizer"
8)
9
10let modelContainer = try await LLMModelFactory.shared.loadContainer(
11    configuration: modelConfiguration
12)
13
14let result = await modelContainer.perform { context in
15    let input = try await context.processor.prepare(
16        input: .init(
17            messages: [
18                ["role": "user", "content": "Summarize this document"]
19            ]
20        )
21    )
22    return try context.model.generate(input, maxTokens: 512)
23}
24
25// iPhone 15 Pro: 18 tokens/sec (A17 Pro Neural Engine)
26// iPhone 14: 12 tokens/sec (A16)

1sudo apt install python3-pip libopenblas-dev libomp-dev
2pip install litert_lm numpy
3
4wget https://huggingface.co/google/gemma-4-e2b/resolve/main/gemma-4-e2b-q8_0.gguf
5
6from fastapi import FastAPI
7from pydantic import BaseModel
8import litert_lm
9
10app = FastAPI()
11interpreter = litert_lm.Interpreter("gemma-4-e2b-q8_0.gguf")
12
13class InferenceRequest(BaseModel):
14    prompt: str
15    max_tokens: int = 256
16
17@app.post("/generate")
18async def generate(request: InferenceRequest):
19    result = interpreter.invoke(
20        prompt=request.prompt,
21        max_tokens=request.max_tokens
22    )
23    return {"text": result, "tokens_per_sec": 8.5}
24

1sudo nano /boot/firmware/config.txt
2arm_freq=2800
3over_voltage=8
4
5sudo nano /etc/default/grub

1import tensorrt as trt
2import pycuda.driver as cuda
3
4builder = trt.Builder(logger)
5network = builder.create_network(
6    1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)
7)
8
9config = builder.create_builder_config()
10config.max_workspace_size = 4 * 1024 * 1024 * 1024  # 4GB
11config.set_flag(trt.BuilderFlag.FP16)
12

1// Transformers.js with WebGPU backend
2import { AutoModelForCausalLM, AutoTokenizer } from '@huggingface/transformers';
3
4const model = await AutoModelForCausalLM.from_pretrained(
5    'google/gemma-4-e2b',
6    { 
7        dtype: 'q4',
8        device: 'webgpu'  // Falls back to WASM if unavailable
9    }
10);
11
12const tokenizer = await AutoTokenizer.from_pretrained('google/gemma-4-e2b');
13
14// Chrome/Edge with WebGPU: 4-6 tokens/sec on integrated graphics
15// Falls back to CPU (WASM) on Safari: 2-3 tokens/sec
16const inputs = await tokenizer('Explain quantum computing in simple terms');
17const outputs = await model.generate(inputs, { max_new_tokens: 256 });

1from vllm import LLM
2
3llm = LLM(
4    model="google/gemma-4-e4b",
5    quantization="awq",
6    kv_cache_dtype="fp8",  # 50% memory reduction
7    max_model_len=16384    # Don't waste cache on unused capacity
8)

1
2config = Gemma4Config(
3    sliding_window=4096,
4    global_attn_every_n_layers=4,
5    max_position_embeddings=32768  # Effective, not absolute
6)

1class ProgressiveLoader:
2    def __init__(self, model_path):
3        self.layer_cache = LRUCache(maxsize=8)  # Keep 8 layers in RAM
4        self.disk_layers = self._index_layers(model_path)
5    
6    def forward(self, layer_idx, hidden_states):
7        if layer_idx not in self.layer_cache:
8            self.layer_cache[layer_idx] = self._load_layer(layer_idx)
9        return self.layer_cache[layer_idx](hidden_states)
10

1
2SYSTEM_PROMPT = """You are a clinical decision support tool. 
3Suggest possible diagnoses based on symptoms, but ALWAYS 
4recommend consulting a physician for confirmation."""
5
6class OfflineMedicalAssistant:
7    def __init__(self):
8        self.model = load_gemma_4_e4b_int8()
9        
10    def triage(self, symptoms: List[str], image: Optional[Image] = None):
11        prompt = f"Symptoms: {', '.join(symptoms)}\nAssess urgency and suggest next steps."
12        
13        if image:
14            return self.model.generate(vision_input=image, text_input=prompt)
15        
16        return self.model.generate(text_input=prompt)

1
2def quality_control_loop():
3    camera = cv2.VideoCapture(0)
4    model = load_gemma_4_e4b_tensorrt()
5    
6    while True:
7        ret, frame = camera.read()
8        if not ret:
9            break
10            
11        result = model.generate(
12            vision_input=frame,
13            text_input="Identify any manufacturing defects. Be specific about type and severity."
14        )
15        
16        if "defect" in result.lower():
17            trigger_alert(result)
18            save_for_review(frame, result)

1
2whisper --model tiny --stream | \
3    python gemma_inference.py | \
4    piper-tts --model en_US-lessac-medium
5