Swift Examples - Replicate

Replicate works slightly differently than the other providers that we support. Most providers allow for a "long http request", essentially one that stays open for the duration of the task. Replicate opted for short http request (which can be a good design for distributed systems) where you get an ID back and then poll for the result. AlProxySwift abstracts this wrinkle away from you and puts it in a convenience method. So under the hood, we are polling for the replicate result every 1 second, which is why you see additional requests with this approach.

When you view requests in the AIProxy live console the difference that you are seeing in the number of requests comes down to the variance that replicate has in its processing. For example, some images take 1.2 seconds, and then the next image will take 5 seconds. We poll every 1 second until replicate tells us that the image is done generating.

You can control the rate limits for replicate by tapping on your replicate service in AlProxy and selecting Global rate limits > Edit

Service setup

Create a Replicate service in the AIProxy dashboard

Follow the integration guide, selecting the Replicate icon on the 'Create a New Service' form.

How to generate a Flux-Schnell image by Black Forest Labs, using Replicate

import AIProxy

let replicateService = AIProxy.replicateService(
    partialKey: "partial-key-from-your-developer-dashboard",
    serviceURL: "service-url-from-your-developer-dashboard"
)

do {
    let input = ReplicateFluxSchnellInputSchema(
        prompt: "Monument valley, Utah"
    )
    let output = try await replicateService.createFluxSchnellImageURLs(
        input: input
    )
    print("Done creating Flux-Schnell image: ", output.first ?? "")
}  catch AIProxyError.unsuccessfulRequest(let statusCode, let responseBody) {
    print("Received \(statusCode) status code with response body: \(responseBody)")
} catch {
    print("Could not create Flux-Schnell image: \(error.localizedDescription)")
}

See the full range of controls for generating an image by viewing ReplicateFluxSchnellInputSchema.swift

How to generate a Flux-Dev image by Black Forest Labs, using Replicate

import AIProxy

let replicateService = AIProxy.replicateService(
    partialKey: "partial-key-from-your-developer-dashboard",
    serviceURL: "service-url-from-your-developer-dashboard"
)

do {
    let input = ReplicateFluxDevInputSchema(
        prompt: "Monument valley, Utah. High res"
    )
    let output = try await replicateService.createFluxDevImageURLs(
        input: input
    )
    print("Done creating Flux-Dev image: ", output.first ?? "")
}  catch AIProxyError.unsuccessfulRequest(let statusCode, let responseBody) {
    print("Received \(statusCode) status code with response body: \(responseBody)")
} catch {
    print("Could not create Flux-Dev image: \(error.localizedDescription)")
}

See the full range of controls for generating an image by viewing ReplicateFluxDevInputSchema.swift

How to generate a Flux-Pro image by Black Forest Labs, using Replicate

This snippet generates a version 1.1 image. If you would like to generate version 1, make the following substitutions:

  • ReplicateFluxProInputSchema_v1_1 -> ReplicateFluxProInputSchema
  • createFluxProImage_v1_1 -> createFluxProImage
import AIProxy

let replicateService = AIProxy.replicateService(
    partialKey: "partial-key-from-your-developer-dashboard",
    serviceURL: "service-url-from-your-developer-dashboard"
)

do {
    let input = ReplicateFluxProInputSchema_v1_1(
        prompt: "Monument valley, Utah. High res"
    )
    let output = try await replicateService.createFluxProImageURL_v1_1(
        input: input
    )
    print("Done creating Flux-Pro image: ", output)
}  catch AIProxyError.unsuccessfulRequest(let statusCode, let responseBody) {
    print("Received \(statusCode) status code with response body: \(responseBody)")
} catch {
    print("Could not create Flux-Pro image: \(error.localizedDescription)")
}

See the full range of controls for generating an image by viewing ReplicateFluxProInputSchema_v1_1.swift

How to generate a Flux-PuLID image using Replicate

On macOS, use NSImage(named:) in place of UIImage(named:)

import AIProxy

let replicateService = AIProxy.replicateService(
    partialKey: "partial-key-from-your-developer-dashboard",
    serviceURL: "service-url-from-your-developer-dashboard"
)

guard let image = UIImage(named: "face") else {
    print("Could not find an image named 'face' in your app assets")
    return
}

guard let imageURL = AIProxy.encodeImageAsURL(image: image, compressionQuality: 0.8) else {
    print("Could not convert image to a local data URI")
    return
}

do {
    let input = ReplicateFluxPulidInputSchema(
        mainFaceImage: imageURL,
        prompt: "smiling man holding sign with glowing green text 'PuLID for FLUX'",
        numOutputs: 1,
        startStep: 4
    )
    let output = try await replicateService.createFluxPulidImage(
        input: input
    )
    print("Done creating Flux-PuLID image: ", output)
}  catch AIProxyError.unsuccessfulRequest(let statusCode, let responseBody) {
    print("Received non-200 status code: \(statusCode) with response body: \(responseBody)")
} catch {
    print("Could not create Flux-Pulid images: \(error.localizedDescription)")
}

How to generate an image from a reference image using Flux ControlNet on Replicate

There are many controls to play with for this use case. Please see ReplicateFluxDevControlNetInputSchema.swift for the full range of controls.

import AIProxy

let replicateService = AIProxy.replicateService(
    partialKey: "partial-key-from-your-developer-dashboard",
    serviceURL: "service-url-from-your-developer-dashboard"
)
do {
    let input = ReplicateFluxDevControlNetInputSchema(
        controlImage: URL(string: "https://example.com/your/image")!,
        prompt: "a cyberpunk with natural greys and whites and browns",
        controlStrength: 0.4
    )
    let output = try await replicateService.createFluxDevControlNetImage(
        input: input
    )
    print("Done creating Flux-ControlNet image: ", output)
}  catch AIProxyError.unsuccessfulRequest(let statusCode, let responseBody) {
    print("Received non-200 status code: \(statusCode) with response body: \(responseBody)")
} catch {
    print("Could not create Flux-ControlNet image: \(error.localizedDescription)")
}

How to generate an SDXL image image by Stability AI, using Replicate

import AIProxy

let replicateService = AIProxy.replicateService(
    partialKey: "partial-key-from-your-developer-dashboard",
    serviceURL: "service-url-from-your-developer-dashboard"
)

do {
    let input = ReplicateSDXLInputSchema(
        prompt: "Monument valley, Utah"
    )
    let output = try await replicateService.createSDXLImage(
        input: input
    )
    print("Done creating SDXL image: ", output.first ?? "")
}  catch AIProxyError.unsuccessfulRequest(let statusCode, let responseBody) {
    print("Received \(statusCode) status code with response body: \(responseBody)")
} catch {
    print("Could not create SDXL image: \(error.localizedDescription)")
}

See the full range of controls for generating an image by viewing ReplicateSDXLInputSchema.swift

How to call your own models on Replicate

  1. Generate the Encodable representation of your input schema. You can use input schemas in this library as inspiration. Take a look at ReplicateFluxProInputSchema.swift as inspiration. Find the schema format that you should conform to using replicate's web dashboard and tapping through Your Model > API > Schema > Input Schema

  2. Generate the Decodable representation of your output schema. For simple cases, a typealias will do (for example, if the output schema is just a string or an array of strings). Look at ReplicateFluxOutputSchema.swift for inspiration. If you need help doing this, please reach out.

  3. Call the createPrediction method, followed by pollForPredictionOutput method. Note that you'll need to change YourInputSchema, YourOutputSchema and your-model-version in this snippet:
import AIProxy

let replicateService = AIProxy.replicateService(
    partialKey: "partial-key-from-your-developer-dashboard",
    serviceURL: "service-url-from-your-developer-dashboard"
)

do {
    let input = YourInputSchema(
        prompt: "Monument valley, Utah"
    )

    let predictionResponse = try await replicateService.createPrediction(
        version: "your-model-version",
        input: input,
        output: ReplicatePredictionResponseBody<YourOutputSchema>.self
    )
    let predictionOutput: YourOutputSchema = try await replicateService.pollForPredictionOutput(
        predictionResponse: predictionResponse,
        pollAttempts: 30
    )
    print("Done creating predictionOutput")
}  catch AIProxyError.unsuccessfulRequest(let statusCode, let responseBody) {
    print("Received \(statusCode) status code with response body: \(responseBody)")
} catch {
    print("Could not create replicate prediction: \(error.localizedDescription)")
}

How to create a replicate model for your own Flux fine tune

Replace <your-account>:

import AIProxy

let replicateService = AIProxy.replicateService(
    partialKey: "partial-key-from-your-developer-dashboard",
    serviceURL: "service-url-from-your-developer-dashboard"
)
do {
    let modelURL = try await replicateService.createModel(owner: "<your-account>", name: "my-model", description: "My great model")
    print("Your model is at \(modelURL)")
}  catch AIProxyError.unsuccessfulRequest(let statusCode, let responseBody) {
    print("Received \(statusCode) status code with response body: \(responseBody)")
} catch {
    print("Could not create replicate model: \(error.localizedDescription)")
}

How to upload training data for your own Flux fine tune

Create a zip file called training.zip and drop it in your Xcode assets. See the "Prepare your training data" section of this guide for tips on what to include in the zip file. Then run:

import AIProxy

let replicateService = AIProxy.replicateService(
    partialKey: "partial-key-from-your-developer-dashboard",
    serviceURL: "service-url-from-your-developer-dashboard"
)

guard let trainingData = NSDataAsset(name: "training") else {
    print("""
          Drop training.zip file into Assets first.
          See the 'Prepare your training data' of this guide:
          https://replicate.com/blog/fine-tune-flux
          """)
    return
}

do {
    let fileUploadResponse = try await replicateService.uploadTrainingZipFile(
        zipData: trainingData.data,
        name: "training.zip"
    )
    print("""
          Training file uploaded. Find it at \(fileUploadResponse.urls.get)
          You you can train with this file until \(fileUploadResponse.expiresAt ?? "")
          """)

}  catch AIProxyError.unsuccessfulRequest(let statusCode, let responseBody) {
    print("Received \(statusCode) status code with response body: \(responseBody)")
} catch {
    print("Could not upload file to replicate: \(error.localizedDescription)")
}

How to train a flux fine-tune

Use the <training-url> returned from the snippet above. Use the <model-name> that you used from the snippet above that. 

import AIProxy

let replicateService = AIProxy.replicateService(
    partialKey: "partial-key-from-your-developer-dashboard",
    serviceURL: "service-url-from-your-developer-dashboard"
)

do {
    // You should experiment with the settings in `ReplicateFluxTrainingInput.swift` to
    // find what works best for your use case.
    //
    // The `layersToOptimizeRegex` argument here speeds training and works well for faces.
    // You could could optionally remove that argument to see if the final trained model
    // works better for your user case.
    let trainingInput = ReplicateFluxTrainingInput(
        inputImages: URL(string: "<training-url>")!,
        layersToOptimizeRegex: "transformer.single_transformer_blocks.(7|12|16|20).proj_out",
        steps: 200,
        triggerWord: "face"
    )
    let reqBody = ReplicateTrainingRequestBody(destination: "<model-owner>/<model-name>", input: trainingInput)


    // Find valid version numbers here: https://replicate.com/ostris/flux-dev-lora-trainer/train
    let training = try await replicateService.createTraining(
        modelOwner: "ostris",
        modelName: "flux-dev-lora-trainer",
        versionID: "d995297071a44dcb72244e6c19462111649ec86a9646c32df56daa7f14801944",
        body: reqBody
    )
    print("Get training status at: \(training.urls?.get?.absoluteString ?? "unknown")")
}  catch AIProxyError.unsuccessfulRequest(let statusCode, let responseBody) {
    print("Received \(statusCode) status code with response body: \(responseBody)")
} catch {
    print("Could not create replicate training: \(error.localizedDescription)")
}

How to poll the flux fine-tune for training complete

Use the <url> that is returned from the snippet above.

import AIProxy

let replicateService = AIProxy.replicateService(
    partialKey: "partial-key-from-your-developer-dashboard",
    serviceURL: "service-url-from-your-developer-dashboard"
)

// This URL comes from the output of the sample above
let url = URL(string: "<url>")!

do {
    let training = try await replicateService.pollForTrainingComplete(
        url: url,
        pollAttempts: 100,
        secondsBetweenPollAttempts: 10
    )
    print("""
          Flux training status: \(training.status?.rawValue ?? "unknown")
          Your model version is: \(training.output?.version ?? "unknown")
          """)
}  catch AIProxyError.unsuccessfulRequest(let statusCode, let responseBody) {
    print("Received \(statusCode) status code with response body: \(responseBody)")
} catch {
    print("Could not poll for the replicate training: \(error.localizedDescription)")
}

How to generate images with your own flux fine-tune

Use the <version> string that was returned from the snippet above, but do not include the model owner and model name in the string.

import AIProxy

let replicateService = AIProxy.replicateService(
    partialKey: "partial-key-from-your-developer-dashboard",
    serviceURL: "service-url-from-your-developer-dashboard"
)

let input = ReplicateFluxFineTuneInputSchema(
    prompt: "an oil painting of my face on a blimp",
    model: .dev,
    numInferenceSteps: 28  // Replicate recommends around 28 steps for `.dev` and 4 for `.schnell`
)

do {
    let predictionResponse = try await replicateService.createPrediction(
        version: "<version>",
        input: input,
        output: ReplicatePredictionResponseBody<[URL]>.self
    )

    let predictionOutput: [URL] = try await replicateService.pollForPredictionOutput(
        predictionResponse: predictionResponse,
        pollAttempts: 30,
        secondsBetweenPollAttempts: 5
    )
    print("Done creating predictionOutput: \(predictionOutput)")
}  catch AIProxyError.unsuccessfulRequest(let statusCode, let responseBody) {
    print("Received \(statusCode) status code with response body: \(responseBody)")
} catch {
    print("Could not create replicate prediction: \(error.localizedDescription)")
}