Converting FastAI Cat vs Dog Model into Core ML

Question

jezekilj OP

Created 1w

Replies 3

Boosts 0

Participants 2

FB:FB16079804

Hello,

I've made the FastAI's Cat vs Dog model into model that distinguishes lemons from limes and it all works fine in a notebook.

I am now looking to transform this model into Core ML for my iOS app using TorchScript and Apple official guidelines for coremltools.

Model converts but I cannot see the Preview Tab in. Xcode. Have anyone of you tried to convert to Core ML? I guess my input types are not matching with coremltools expectations for preview but I am stuck . Here is my code.

import torch
import coremltools as ct
from fastai.vision.all import *
import json
from torchvision import transforms

# Load your Fastai model (replace with your actual path)
learn = load_learner('lemonmodel.pkl')

# Example input image (you can use any image from your dataset)
input_image = PILImage.create('example.jpg')

# Preprocess the image (assuming you used these transforms during training)
to_tensor = transforms.ToTensor()
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
input_tensor = to_tensor(input_image)
input_tensor = normalize(input_tensor)  # Apply normalization

# Add a batch dimension
input_tensor = input_tensor.unsqueeze(0)

# Ensure float32 type
input_tensor = input_tensor.float()

# Trace the model
trace = torch.jit.trace(learn.model, input_tensor)

# Define the Core ML input type (considering your model's input shape)
_input = ct.ImageType(
    name="input_1",
    shape=input_tensor.shape,
    bias=[-0.485/0.229, -0.456/0.224, -0.406/0.225],
    scale=1./(255*0.226)
)

# Convert the model to Core ML format
mlmodel = ct.convert(
    trace,
    inputs=[_input],
    minimum_deployment_target=ct.target.iOS14  # Optional, set deployment target
)

# Set model type as 'imageClassifier' for the Preview tab
mlmodel.type = 'imageClassifier'

# Correct structure for preview parameters** (assuming two classes: 'lemon' and 'lime')
labels_json = {
    "imageClassifier": {
        "labels": ["lemon", "lime"],
        "input": {
            "shape": list(input_tensor.shape),  # Provide the actual input shape
            "mean": [0.485, 0.456, 0.406],  # Match normalization mean
            "std": [0.229, 0.224, 0.225]   # Match normalization std
        },
        "output": {
            "shape": [1, 2]  # Output shape for your model (2 classes)
        }
    }
}

# Setting up the metadata with correct 'preview' params
mlmodel.user_defined_metadata['com.apple.coreml.model.preview.params'] = json.dumps(labels_json)

# Save the model as .mlmodel
mlmodel.save("LemonClassifierGemini.mlmodel")
mlmodel = ct.convert(
    trace,
    inputs=[_input],
    minimum_deployment_target=ct.target.iOS14  # Optional, set deployment target
)

# Set model type as 'imageClassifier' for the Preview tab**
mlmodel.type = 'imageClassifier'

# Correct structure for preview parameters** (assuming two classes: 'lemon' and 'lime')
labels_json = {
    "imageClassifier": {
        "labels": ["lemon", "lime"],
        "input": {
            "shape": list(input_tensor.shape),  # Provide the actual input shape
            "mean": [0.485, 0.456, 0.406],  # Match normalization mean
            "std": [0.229, 0.224, 0.225]   # Match normalization std
        },
        "output": {
            "shape": [1, 2]  # Output shape for your model (2 classes)
        }
    }
}

# Setting up the metadata with correct 'preview' params**
mlmodel.user_defined_metadata['com.apple.coreml.model.preview.params'] = json.dumps(labels_json)

# Save the model as .mlmodel
mlmodel.save("LemonClassifierGemini.mlmodel")

My model is :

Input batch shape: torch.Size([32, 3, 192, 192]) Labels batch shape: torch.Size([32])

Validation Loss: None, Validation Metric: None

Predictions shape: torch.Size([63, 2]) Targets shape: torch.Size([63])

Code for the model :

searches = 'lemon','lime'
path = Path('lemon_or_not')

for o in searches:
    dest = (path/o)
    dest.mkdir(exist_ok=True, parents=True)
    download_images(dest, urls=search_images(f'{o} photo'))
    time.sleep(5)
    resize_images(path/o, max_size=400, dest=path/o)

dls = DataBlock(
    blocks=(ImageBlock, CategoryBlock), 
    get_items=get_image_files, 
    splitter=RandomSplitter(valid_pct=0.2, seed=42),
    get_y=parent_label,
    item_tfms=[Resize(192, method='squish')]
).dataloaders(path, bs=32)

dls.show_batch(max_n=6)

learn = vision_learner(dls, resnet18, metrics=error_rate)
learn.fine_tune(3)

is_lemon,_,probs = learn.predict(PILImage.create('lemon.jpg'))
print(f"This is a: {is_lemon}.")
print(f"Probability it's a lemon: {probs[0]:.4f}")

This is a: lemon. Probability it's a lemon: 1.0000

learn.export('lemonmodel.pkl')

I am stuck to why it doest show the Preview Tab.

Answered by Frameworks Engineer in 817642022

I am glad you have solved the issue. Thank you for the update!

Boost

Answer 1

jezekilj OP

1w

I was able to get the results with this code and with preview working but:

in preview it says "unable to retrieve results from the vision request".

import torch
import coremltools as ct
from fastai.vision.all import *
import json
from PIL import Image
from torchvision import transforms

# Load your Fastai model (replace with your actual path)
learn = load_learner('lemonmodel.pkl')

# Set the model to eval mode before tracing
learn.model.eval()

# Example input image (you can use any image from your dataset)
input_image = PILImage.create('example.jpg')

# Preprocess the image (assuming you used these transforms during training)
resize = transforms.Resize((192, 192))  # Resize image to match model input size (e.g., 192x192)
to_tensor = transforms.ToTensor()
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])

# Apply preprocessing: Resize, Convert to tensor, Normalize
input_image = resize(input_image)  # Resize to expected size
input_tensor = to_tensor(input_image)  # Convert to tensor
input_tensor = normalize(input_tensor)  # Normalize with mean and std

# Add a batch dimension
input_tensor = input_tensor.unsqueeze(0)

# Ensure float32 type
input_tensor = input_tensor.float()

# Trace the model (using the batch-size of 1)
trace = torch.jit.trace(learn.model, input_tensor)

# Define the Core ML input type (image type with correct shape for Core ML)
_input = ct.ImageType(
    name="input_1",
    shape=(1, 3, 192, 192),  # Correct shape for Core ML [batch_size, channels, height, width]
    bias=[-0.485 / 0.229, -0.456 / 0.224, -0.406 / 0.225],  # Mean normalization
    scale=1.0 / 255  # Scale normalization
)

# Define the Core ML output type (we do NOT specify the shape, let Core ML infer it)
_output = ct.TensorType(
    name="output_1",  # Name for the output
)

# Convert the model to Core ML format
mlmodel = ct.convert(
    trace,
    inputs=[_input],
    outputs=[_output],  # Let Core ML infer the output shape
    minimum_deployment_target=ct.target.iOS14  # iOS deployment target
)

# Set model type as 'imageClassifier' for the Preview tab
mlmodel.type = 'imageClassifier'

# Define labels for classification
labels_json = {
    "labels": ["lemon", "lime"],  # Replace with your actual class labels
}

# Setting up the metadata with correct 'preview' params
mlmodel.user_defined_metadata['com.apple.coreml.model.preview.params'] = json.dumps(labels_json)

# Set model metadata for Xcode integration
mlmodel.user_defined_metadata["com.apple.coreml.model.preview.type"] = "imageClassifier"
mlmodel.input_description["input_1"] = "Input image to be classified"
mlmodel.output_description["output_1"] = "Classification probabilities for each label"

# Set additional metadata for the Xcode UI (optional)
mlmodel.author = "Your Name or Organization"
mlmodel.short_description = "A classifier for detecting lemon and lime in images."
mlmodel.version = "1.0"

# Save the model as .mlmodel
mlmodel.save("LemonClassifier333.mlmodel")

0

Answer 2

jezekilj OP

1w

I've fixed the issue and published it to iOS. After reviewing the documentation and source code for coremltools v8.1, I realised that I didn't need to specify the outputs – coremltools automatically handles it. 😁

Screenshot 2024-12-11 at 14.13.42.jpeg

0

Answer 3

Frameworks Engineer OP

Apple

1w

Accepted Answer

I am glad you have solved the issue. Thank you for the update!

1