Microsoft HoloLens has a variety of sensors, making it a very powerful tool. The device is placed on the head and you can view the space from their point of view. This guide represents how you can take a photo via HoloLens and project it on a holographic surface and how you can record and play videos.

Initially, create a new scene with the name “Capture-Play-Photo-Video”. It is important to set the Camera of the scene, the projection to Perspective. Also, you need to be in the right position as shown in the picture. Position (0, -0.02, 0), Rotation (0, 0, 0), Scale (1, 1, 1). Most importantly, the point where you choose to display the camera is the point based on which the other objects in space will be initialized.

Add two new GameObjects (Cube) in the unity scene under the names “DisplayPhoto” and “PlayVideo” and give the following transform to the images.

 

Add a new GameObject named “CaptureManager” which will contain the scripts, KeywordGlobalManager.cs which enables Capture Photo, Start Recording, Stop Recording, PhotoCaptureHandler.cs that manages the capture photo mode, RecordVideoHandler.cs that manages the record video mode and the UIManager.cs script which displays the photo on GameObject “DisplaysPhoto” and plays the video on GameObject “PlayVideo”.

PhotoCaptureHandler.cs

public class PhotoCaptureHandler : MonoBehaviour
{
    PhotoCapture photoCapture = null;

    public void StartPhotoCapture()
    {
        PhotoCapture.CreateAsync(false, OnPhotoCaptureCreated);
    }
    
    void OnPhotoCaptureCreated(PhotoCapture captureObject)
    {
        photoCapture = captureObject;

        Resolution cameraResolution = PhotoCapture.SupportedResolutions.OrderByDescending((res) => res.width * res.height).First();

        CameraParameters c = new CameraParameters();
        c.hologramOpacity = 0.0f;
        c.cameraResolutionWidth = cameraResolution.width;
        c.cameraResolutionHeight = cameraResolution.height;
        c.pixelFormat = CapturePixelFormat.BGRA32;

        captureObject.StartPhotoModeAsync(c, OnPhotoModeStarted);
    }

    private void OnStoppedPhotoMode(PhotoCapture.PhotoCaptureResult result)
    {
        photoCapture.Dispose();
        photoCapture = null;
    }

    private void OnPhotoModeStarted(PhotoCapture.PhotoCaptureResult result)
    {
        if (result.success)
        {
            try
            {
                photoCapture.TakePhotoAsync(OnCapturedPhotoToMemory);
            }
            catch (System.ArgumentException e)
            {
                Debug.LogError("System.ArgumentException:\n" + e.Message);
            }
        }
        else
        {
            Debug.LogError("Unable to start photo mode!");
        }
    }

    void OnCapturedPhotoToDisk(PhotoCapture.PhotoCaptureResult result)
    {
        if (result.success)
        {
            Debug.Log("Saved Photo to disk!");
            photoCapture.StopPhotoModeAsync(OnStoppedPhotoMode);
        }
        else
        {
            Debug.Log("Failed to save Photo to disk");
        }
    }

    void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
    {
        if (result.success)
        {
            List<byte> imageBufferList = new List<byte>();

            Debug.Log("OnCapturedPhotoToMemory Copy Started");

            photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);

            Debug.Log("OnCapturedPhotoToMemory " + imageBufferList.Count);
            
            Resolution cameraResolution = PhotoCapture.SupportedResolutions.OrderByDescending((res) => res.width * res.height).First();
            Texture2D texture = new Texture2D(cameraResolution.width, cameraResolution.height);
            photoCaptureFrame.UploadImageDataToTexture(texture);

            UIManager.Instance.LoadPhotoToTexture(texture);
        }
        else
        {
            Debug.Log("Failed to save Photo to memory");
        }

        photoCapture.StopPhotoModeAsync(OnStoppedPhotoMode);
    }
}

RecordVideoHandler.cs

public class RecordVideoHandler : MonoBehaviour {

    VideoCapture m_VideoCapture = null;

    public void StartRecordingVideo()
    {
        VideoCapture.CreateAsync(false, OnVideoCaptureCreated);
    }

    public void StopRecordingVideo()
    {
        m_VideoCapture.StopRecordingAsync(OnStoppedRecordingVideo);
    }

    void OnVideoCaptureCreated(VideoCapture videoCapture)
    {
        if (videoCapture != null)
        {
            m_VideoCapture = videoCapture;

            Resolution cameraResolution = VideoCapture.SupportedResolutions.OrderByDescending((res) => res.width * res.height).First();
            float cameraFramerate = VideoCapture.GetSupportedFrameRatesForResolution(cameraResolution).OrderByDescending((fps) => fps).First();

            CameraParameters cameraParameters = new CameraParameters();
            cameraParameters.hologramOpacity = 0.0f;
            cameraParameters.frameRate = cameraFramerate;
            cameraParameters.cameraResolutionWidth = cameraResolution.width;
            cameraParameters.cameraResolutionHeight = cameraResolution.height;
            cameraParameters.pixelFormat = CapturePixelFormat.BGRA32;

            m_VideoCapture.StartVideoModeAsync(cameraParameters,
                                                VideoCapture.AudioState.None,
                                                OnStartedVideoCaptureMode);
        }
        else
        {
            Debug.LogError("Failed to create VideoCapture Instance!");
        }
    }

    void OnStartedVideoCaptureMode(VideoCapture.VideoCaptureResult result)
    {
        if (result.success)
        {
            string filename = string.Format("MyVideo_{0}.mp4", Time.time);
            string filepath = System.IO.Path.Combine(Application.persistentDataPath, filename);

            m_VideoCapture.StartRecordingAsync(filepath, OnStartedRecordingVideo);
        }
    }

    void OnStartedRecordingVideo(VideoCapture.VideoCaptureResult result)
    {
        Debug.Log("Started Recording Video!");
    }

    void OnStoppedRecordingVideo(VideoCapture.VideoCaptureResult result)
    {
        Debug.Log("Stopped Recording Video!");
        m_VideoCapture.StopVideoModeAsync(OnStoppedVideoCaptureMode);
    }

    void OnStoppedVideoCaptureMode(VideoCapture.VideoCaptureResult result)
    {
        m_VideoCapture.Dispose();
        m_VideoCapture = null;
    }
}

UIManager.cs

public class UIManager : MonoBehaviour {

    public static UIManager Instance { get; private set; }

    public GameObject gameObjDisplayPhoto;
    public GameObject gameObjPlayVideo;
    void Awake()
    {
        Instance = this;
    }

    public void LoadPhotoToTexture(Texture2D texture)
    {
        gameObjDisplayPhoto.GetComponent<MeshRenderer>().materials[0].SetTexture("_MainTex", texture);
    }

    public void PlayVideo()
    {
        ((MovieTexture)gameObjPlayVideo.GetComponent<Renderer>().material.mainTexture).Play();
    }
}

Finally, you ‘ll need to enable some capabilities in order to have access to camera, sound etc. Choose in unity Edit->Project Settings->Player. From the “Publishing settings” tab in the capabilities section check, “Video Library”, “WebCam”, “Microphone”.

To capture and display the photo use the voice command “Capture Photo”, to start recording video “Start Recording” command and to stop recording the video and play on holographic surface “Stop Recording” command.

You can find this sample and download it from github to this link https://github.com/gntakakis/Hololens-Capture-Play-Photo-Video-Unity.