Mobile AI Development: How to Run AI Models on Devices

 This video shows how you can install and run LLMs on smart devices like mobile phone, tablets, IOT devices, drones, cameras, smart watches etc by using Qualcomm AI Hub.

Code:

pip install torch

pip install transformers

pip install ipywidgets

pip install psutil

pip install huggingface_hub

pip install pillow

pip install qai-hub

pip install qai_hub_models

pip install "qai_hub_models[yolov7]"

huggingface-cli login

qai-hub configure --api_token <token>

qai-hub list-devices

from PIL import Image

from typing import Tuple

import torch

import qai_hub as hub

from qai_hub_models.models.yolov7 import Model as YOLOv7Model

# Load YOLOv7 in PyTorch

torch_model = YOLOv7Model.from_pretrained()

torch_model.eval()

# Trace the PyTorch model using one data point of provided sample inputs to

# torch tensor to trace the model.

example_input = [torch.tensor(data[0]) for name, data in torch_model.sample_inputs().items()]

pt_model = torch.jit.trace(torch_model, example_input)

# Select a device

device = hub.Device("Samsung Galaxy S23")

# Compile model for a specific device

compile_job = hub.submit_compile_job(

    model=pt_model,

    device=device,

    input_specs=torch_model.get_input_spec(),

)

# Get target model to run on a cloud hosted device

target_model = compile_job.get_target_model()

# Profile the previously compiled model on a cloud hosted device

profile_job = hub.submit_profile_job(

    model=target_model,

    device=device,

)

# Perform on-device inference on a cloud hosted device

input_data = torch_model.sample_inputs()

inference_job = hub.submit_inference_job(

    model=target_model,

    device=device,

    inputs=input_data,

)

# Returns the output as dict{name: numpy}

on_device_output = inference_job.download_output_data()

print(on_device_output)