2021 Graphics & GamesSpatial Computing
WWDC21 · 25 min · Graphics & Games / Spatial Computing
Dive into RealityKit 2
Creating engaging AR experiences has never been easier with RealityKit 2. Explore the latest enhancements to the RealityKit framework and take a deep dive into this underwater sample project. We’ll take you through the improved Entity Component System, streamlined animation pipeline, and the plug-and-play character controller with enhancements to face mesh and audio.
Watch at developer.apple.com ↗Code shown on screen · 7 snippets
FlockingSystem
class FlockingSystem: RealityKit.System {
required init(scene: RealityKit.Scene) { }
static var dependencies: [SystemDependency] { [.before(MotionSystem.self)] }
private static let query = EntityQuery(where: .has(FlockingComponent.self)
&& .has(MotionComponent.self)
&& .has(SettingsComponent.self)) FlockingSystem.update
func update(context: SceneUpdateContext) {
context.scene.performQuery(Self.query).forEach { entity in
guard var motion: MotionComponent = entity.components[MotionComponent.self]
else { continue }
// ... Using a Boids simulation, add forces to the MotionComponent
motion.forces.append(/* separation, cohesion, alignment forces */)
entity.components[MotionComponent.self] = motion
}
} Store Subscription While Entity Active
arView.scene.subscribe(to: CollisionEvents.Began.self, on: fish) { [weak self] event in
// ... handle collisions with this particular fish
}.storeWhileEntityActive(fish) SwiftUI + RealityKit Settings Instance
class Settings: ObservableObject {
var separationWeight: Float = 1.6
// ...
}
struct ContentView : View {
var settings = Settings()
var body: some View {
ZStack {
ARViewContainer(settings: settings)
MovementSettingsView()
.environmentObject(settings)
}
}
}
struct SettingsComponent: RealityKit.Component {
var settings: Settings
}
class UnderwaterView: ARView {
let settings: Settings
private func addEntity(_ entity: Entity) {
entity.components[SettingsComponent.self] =
SettingsComponent(settings: self.settings)
}
} FaceMesh
static let sceneUnderstandingQuery =
EntityQuery(where: .has(SceneUnderstandingComponent.self) && .has(ModelComponent.self))
func findFaceEntity(scene: RealityKit.Scene) -> HasModel? {
let faceEntity = scene.performQuery(sceneUnderstandingQuery).first {
$0.components[SceneUnderstandingComponent.self]?.entityType == .face
}
return faceEntity as? HasModel
} FaceMesh - Painting material
func updateFaceEntityTextureUsing(cgImage: CGImage) {
guard let faceEntity = self.faceEntity else { return }
guard let faceTexture =
try? TextureResource.generate(from: cgImage,
options: .init(semantic: .color))
else { return }
var faceMaterial = PhysicallyBasedMaterial()
faceMaterial.roughness = 0.1
faceMaterial.metallic = 1.0
faceMaterial.blending = .transparent(opacity: .init(scale: 1.0))
let sparklyNormalMap = try! TextureResource.load(named: "sparkly")
faceMaterial.normal.texture = PhysicallyBasedMaterial.Texture.init(sparklyNormalMap)
faceMaterial.baseColor.texture = PhysicallyBasedMaterial.Texture.init(faceTexture)
faceEntity.model!.materials = [faceMaterial]
} AudioBufferResource
let synthesizer = AVSpeechSynthesizer()
func speakText(_ text: String, forEntity entity: Entity) {
let utterance = AVSpeechUtterance(string: text)
utterance.voice = AVSpeechSynthesisVoice(language: "en-IE")
synthesizer.write(utterance) { audioBuffer in
guard
let audioResource = try? AudioBufferResource(buffer: audioBuffer,
inputMode: .spatial,
shouldLoop: true)
else { return }
entity.playAudio(audioResource)
}
} Resources
Related sessions
-
34 min -
26 min -
20 min -
30 min -
25 min -
37 min