From 6cfb6599a5b9146dfa9ebfb4c5ad334b7eb0bdbb Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 7 Oct 2015 10:31:42 +0200 Subject: [PATCH 1/9] HoloLens scanning effect --- .../ProjectionMappingSample/App.config | 32 +- .../CameraDeviceResource.cs | 333 +++++++++ .../DepthAndColorShader.cs | 4 - .../ProjectionMappingSample/Form1.cs | 11 +- .../ProjectionMappingSample.cs | 648 +----------------- .../ProjectionMappingSample.csproj | 1 + .../ProjectiveTexturingShader.cs | 19 +- .../Properties/Settings.Designer.cs | 50 +- .../Properties/Settings.settings | 16 +- .../Shaders/DepthAndColorGS.hlsl | 2 + .../Shaders/DepthAndColorPS.hlsl | 106 ++- .../Shaders/DepthAndProjectiveTextureVS.hlsl | 4 + 12 files changed, 492 insertions(+), 734 deletions(-) create mode 100644 ProCamCalibration/ProjectionMappingSample/CameraDeviceResource.cs diff --git a/ProCamCalibration/ProjectionMappingSample/App.config b/ProCamCalibration/ProjectionMappingSample/App.config index 272bc19..2223e7c 100644 --- a/ProCamCalibration/ProjectionMappingSample/App.config +++ b/ProCamCalibration/ProjectionMappingSample/App.config @@ -8,26 +8,14 @@ - - - - True - - - False - - - False - - - False - - - False - - - False - - - + + + + True + + + True + + + \ No newline at end of file diff --git a/ProCamCalibration/ProjectionMappingSample/CameraDeviceResource.cs b/ProCamCalibration/ProjectionMappingSample/CameraDeviceResource.cs new file mode 100644 index 0000000..9a7a140 --- /dev/null +++ b/ProCamCalibration/ProjectionMappingSample/CameraDeviceResource.cs @@ -0,0 +1,333 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using SharpDX; +using SharpDX.Direct3D11; +using SharpDX.DXGI; +using SharpDX.WIC; +using System.IO; + + +namespace RoomAliveToolkit +{ + class CameraDeviceResource : IDisposable + { + // encapsulates d3d resources for a camera + public CameraDeviceResource(SharpDX.Direct3D11.Device device, ProjectorCameraEnsemble.Camera camera, Object renderLock, string directory) + { + this.device = device; + this.camera = camera; + this.renderLock = renderLock; + + // Kinect depth image + var depthImageTextureDesc = new Texture2DDescription() + { + Width = Kinect2Calibration.depthImageWidth, + Height = Kinect2Calibration.depthImageHeight, + MipLevels = 1, + ArraySize = 1, + Format = SharpDX.DXGI.Format.R16_UInt, + SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), + Usage = ResourceUsage.Dynamic, + BindFlags = BindFlags.ShaderResource, + CpuAccessFlags = CpuAccessFlags.Write, + }; + depthImageTexture = new Texture2D(device, depthImageTextureDesc); + depthImageTextureRV = new ShaderResourceView(device, depthImageTexture); + + var floatDepthImageTextureDesc = new Texture2DDescription() + { + Width = Kinect2Calibration.depthImageWidth, + Height = Kinect2Calibration.depthImageHeight, + MipLevels = 1, + ArraySize = 1, + Format = SharpDX.DXGI.Format.R32_Float, + SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), + Usage = ResourceUsage.Default, + BindFlags = BindFlags.RenderTarget | BindFlags.ShaderResource, + CpuAccessFlags = CpuAccessFlags.None, + }; + + floatDepthImageTexture = new Texture2D(device, floatDepthImageTextureDesc); + floatDepthImageRV = new ShaderResourceView(device, floatDepthImageTexture); + floatDepthImageRenderTargetView = new RenderTargetView(device, floatDepthImageTexture); + + floatDepthImageTexture2 = new Texture2D(device, floatDepthImageTextureDesc); + floatDepthImageRV2 = new ShaderResourceView(device, floatDepthImageTexture2); + floatDepthImageRenderTargetView2 = new RenderTargetView(device, floatDepthImageTexture2); + + // Kinect color image + var colorImageStagingTextureDesc = new Texture2DDescription() + { + Width = Kinect2Calibration.colorImageWidth, + Height = Kinect2Calibration.colorImageHeight, + MipLevels = 1, + ArraySize = 1, + Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, + SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), + Usage = ResourceUsage.Dynamic, + BindFlags = BindFlags.ShaderResource, + CpuAccessFlags = CpuAccessFlags.Write + }; + colorImageStagingTexture = new Texture2D(device, colorImageStagingTextureDesc); + + var colorImageTextureDesc = new Texture2DDescription() + { + Width = Kinect2Calibration.colorImageWidth, + Height = Kinect2Calibration.colorImageHeight, + MipLevels = 0, + ArraySize = 1, + Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, + SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), + Usage = ResourceUsage.Default, + BindFlags = BindFlags.ShaderResource | BindFlags.RenderTarget, + CpuAccessFlags = CpuAccessFlags.None, + OptionFlags = ResourceOptionFlags.GenerateMipMaps + }; + colorImageTexture = new Texture2D(device, colorImageTextureDesc); + colorImageTextureRV = new ShaderResourceView(device, colorImageTexture); + + // vertex buffer + var table = camera.calibration.ComputeDepthFrameToCameraSpaceTable(); + int numVertices = 6 * (Kinect2Calibration.depthImageWidth - 1) * (Kinect2Calibration.depthImageHeight - 1); + var vertices = new VertexPosition[numVertices]; + + Int3[] quadOffsets = new Int3[] + { + new Int3(0, 0, 0), + new Int3(1, 0, 0), + new Int3(0, 1, 0), + new Int3(1, 0, 0), + new Int3(1, 1, 0), + new Int3(0, 1, 0), + }; + + int vertexIndex = 0; + for (int y = 0; y < Kinect2Calibration.depthImageHeight - 1; y++) + { + for (int x = 0; x < Kinect2Calibration.depthImageWidth - 1; x++) + { + for (int i = 0; i < 6; i++) + { + int vertexX = x + quadOffsets[i].X; + int vertexY = y + quadOffsets[i].Y; + + var point = table[Kinect2Calibration.depthImageWidth * vertexY + vertexX]; + + var vertex = new VertexPosition(); + vertex.position = new SharpDX.Vector4(point.X, point.Y, vertexX, vertexY); + vertices[vertexIndex++] = vertex; + } + } + } + + var stream = new DataStream(numVertices * VertexPosition.SizeInBytes, true, true); + stream.WriteRange(vertices); + stream.Position = 0; + + var vertexBufferDesc = new BufferDescription() + { + BindFlags = BindFlags.VertexBuffer, + CpuAccessFlags = CpuAccessFlags.None, + Usage = ResourceUsage.Default, + SizeInBytes = numVertices * VertexPosition.SizeInBytes, + }; + vertexBuffer = new SharpDX.Direct3D11.Buffer(device, stream, vertexBufferDesc); + + vertexBufferBinding = new VertexBufferBinding(vertexBuffer, VertexPosition.SizeInBytes, 0); + + stream.Dispose(); + + //var colorImage = new RoomAliveToolkit.ARGBImage(Kinect2Calibration.colorImageWidth, Kinect2Calibration.colorImageHeight); + //ProjectorCameraEnsemble.LoadFromTiff(imagingFactory, colorImage, directory + "/camera" + camera.name + "/color.tiff"); + + var depthImage = new RoomAliveToolkit.ShortImage(Kinect2Calibration.depthImageWidth, Kinect2Calibration.depthImageHeight); + ProjectorCameraEnsemble.LoadFromTiff(imagingFactory, depthImage, directory + "/camera" + camera.name + "/mean.tiff"); + + lock (renderLock) // necessary? + { + //UpdateColorImage(device.ImmediateContext, colorImage.DataIntPtr); + UpdateDepthImage(device.ImmediateContext, depthImage.DataIntPtr); + } + + //colorImage.Dispose(); + depthImage.Dispose(); + } + + struct VertexPosition + { + public SharpDX.Vector4 position; + static public int SizeInBytes { get { return 4 * 4; } } + } + + public void Dispose() + { + depthImageTexture.Dispose(); + depthImageTextureRV.Dispose(); + colorImageTexture.Dispose(); + colorImageTextureRV.Dispose(); + colorImageStagingTexture.Dispose(); + vertexBuffer.Dispose(); + } + + SharpDX.Direct3D11.Device device; + public Texture2D depthImageTexture, floatDepthImageTexture, floatDepthImageTexture2; + public ShaderResourceView depthImageTextureRV, floatDepthImageRV, floatDepthImageRV2; + public RenderTargetView floatDepthImageRenderTargetView, floatDepthImageRenderTargetView2; + public Texture2D colorImageTexture; + public ShaderResourceView colorImageTextureRV; + public Texture2D colorImageStagingTexture; + public SharpDX.Direct3D11.Buffer vertexBuffer; + VertexBufferBinding vertexBufferBinding; + ProjectorCameraEnsemble.Camera camera; + public bool renderEnabled = true; + + public void UpdateDepthImage(DeviceContext deviceContext, IntPtr depthImage) + { + DataStream dataStream; + deviceContext.MapSubresource(depthImageTexture, 0, + MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); + dataStream.WriteRange(depthImage, Kinect2Calibration.depthImageWidth * Kinect2Calibration.depthImageHeight * 2); + deviceContext.UnmapSubresource(depthImageTexture, 0); + } + + public void UpdateDepthImage(DeviceContext deviceContext, byte[] depthImage) + { + DataStream dataStream; + deviceContext.MapSubresource(depthImageTexture, 0, + MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); + dataStream.WriteRange(depthImage, 0, Kinect2Calibration.depthImageWidth * Kinect2Calibration.depthImageHeight * 2); + deviceContext.UnmapSubresource(depthImageTexture, 0); + } + + public void UpdateColorImage(DeviceContext deviceContext, IntPtr colorImage) + { + DataStream dataStream; + deviceContext.MapSubresource(colorImageStagingTexture, 0, + MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); + dataStream.WriteRange(colorImage, Kinect2Calibration.colorImageWidth * Kinect2Calibration.colorImageHeight * 4); + deviceContext.UnmapSubresource(colorImageStagingTexture, 0); + + var resourceRegion = new ResourceRegion() + { + Left = 0, + Top = 0, + Right = Kinect2Calibration.colorImageWidth, + Bottom = Kinect2Calibration.colorImageHeight, + Front = 0, + Back = 1, + }; + deviceContext.CopySubresourceRegion(colorImageStagingTexture, 0, resourceRegion, colorImageTexture, 0); + deviceContext.GenerateMips(colorImageTextureRV); + } + + public void UpdateColorImage(DeviceContext deviceContext, byte[] colorImage) + { + DataStream dataStream; + deviceContext.MapSubresource(colorImageStagingTexture, 0, + MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); + dataStream.WriteRange(colorImage, 0, Kinect2Calibration.colorImageWidth * Kinect2Calibration.colorImageHeight * 4); + deviceContext.UnmapSubresource(colorImageStagingTexture, 0); + + var resourceRegion = new ResourceRegion() + { + Left = 0, + Top = 0, + Right = Kinect2Calibration.colorImageWidth, + Bottom = Kinect2Calibration.colorImageHeight, + Front = 0, + Back = 1, + }; + deviceContext.CopySubresourceRegion(colorImageStagingTexture, 0, resourceRegion, colorImageTexture, 0); + deviceContext.GenerateMips(colorImageTextureRV); + } + + public void Render(DeviceContext deviceContext) + { + deviceContext.InputAssembler.SetVertexBuffers(0, vertexBufferBinding); + deviceContext.VertexShader.SetShaderResource(0, depthImageTextureRV); + deviceContext.PixelShader.SetShaderResource(0, colorImageTextureRV); + deviceContext.Draw((Kinect2Calibration.depthImageWidth - 1) * (Kinect2Calibration.depthImageHeight - 1) * 6, 0); + } + + bool live = false; + + public void StartLive() + { + live = true; + //new System.Threading.Thread(ColorCameraLoop).Start(); + new System.Threading.Thread(DepthCameraLoop).Start(); + } + + public void StopLive() + { + live = false; + } + + + Object renderLock; + public bool depthImageChanged = true; + + //byte[] colorData = new byte[4 * Kinect2.Kinect2Calibration.colorImageWidth * Kinect2.Kinect2Calibration.colorImageHeight]; + byte[] nextColorData = new byte[4 * RoomAliveToolkit.Kinect2Calibration.colorImageWidth * RoomAliveToolkit.Kinect2Calibration.colorImageHeight]; + SharpDX.WIC.ImagingFactory2 imagingFactory = new SharpDX.WIC.ImagingFactory2(); + void ColorCameraLoop() + { + while (true) + { + var encodedColorData = camera.Client.LatestJPEGImage(); + + // decode JPEG + var memoryStream = new MemoryStream(encodedColorData); + var stream = new WICStream(imagingFactory, memoryStream); + // decodes to 24 bit BGR + var decoder = new SharpDX.WIC.BitmapDecoder(imagingFactory, stream, SharpDX.WIC.DecodeOptions.CacheOnLoad); + var bitmapFrameDecode = decoder.GetFrame(0); + + // convert to 32 bpp + var formatConverter = new FormatConverter(imagingFactory); + formatConverter.Initialize(bitmapFrameDecode, SharpDX.WIC.PixelFormat.Format32bppBGR); + formatConverter.CopyPixels(nextColorData, Kinect2Calibration.colorImageWidth * 4); // TODO: consider copying directly to texture native memory + //lock (colorData) + // Swap(ref colorData, ref nextColorData); + lock (renderLock) // necessary? + { + UpdateColorImage(device.ImmediateContext, nextColorData); + } + memoryStream.Close(); + memoryStream.Dispose(); + stream.Dispose(); + decoder.Dispose(); + formatConverter.Dispose(); + bitmapFrameDecode.Dispose(); + } + } + + //byte[] depthData = new byte[2 * Kinect2.Kinect2Calibration.depthImageWidth * Kinect2.Kinect2Calibration.depthImageHeight]; + byte[] nextDepthData; + void DepthCameraLoop() + { + while (true) + { + nextDepthData = camera.Client.LatestDepthImage(); + //lock (remoteDepthData) + // Swap(ref remoteDepthData, ref nextRemoteDepthData); + lock (renderLock) + { + depthImageChanged = true; + UpdateDepthImage(device.ImmediateContext, nextDepthData); + } + } + } + + static void Swap(ref T first, ref T second) + { + T temp = first; + first = second; + second = temp; + } + } +} diff --git a/ProCamCalibration/ProjectionMappingSample/DepthAndColorShader.cs b/ProCamCalibration/ProjectionMappingSample/DepthAndColorShader.cs index 1a777aa..f13eb8e 100644 --- a/ProCamCalibration/ProjectionMappingSample/DepthAndColorShader.cs +++ b/ProCamCalibration/ProjectionMappingSample/DepthAndColorShader.cs @@ -293,12 +293,8 @@ public void Render(DeviceContext deviceContext, ShaderResourceView depthImageTex SamplerState colorSamplerState; SharpDX.Direct3D11.Buffer constantBuffer; BilateralFilter bilateralFilter; - FromUIntPS fromUIntPS; Texture2D filteredDepthImageTexture, filteredDepthImageTexture2; RenderTargetView filteredRenderTargetView, filteredRenderTargetView2; ShaderResourceView filteredDepthImageSRV, filteredDepthImageSRV2; } - - - } diff --git a/ProCamCalibration/ProjectionMappingSample/Form1.cs b/ProCamCalibration/ProjectionMappingSample/Form1.cs index 165b6bb..1a40a92 100644 --- a/ProCamCalibration/ProjectionMappingSample/Form1.cs +++ b/ProCamCalibration/ProjectionMappingSample/Form1.cs @@ -133,7 +133,16 @@ public bool FullScreen // switch to fullscreen ShowInTaskbar = false; FormBorderStyle = FormBorderStyle.None; - var bounds = Screen.AllScreens[projector.displayIndex].Bounds; // TODO: catch the case where the display is not available + + // Clamp to last screen if screen number is not available + if (projector.displayIndex >= Screen.AllScreens.Length) + { + Console.WriteLine("Projector '{0}' Clamped displayIndex '{1}' to '{2}'", projector.name, projector.displayIndex, Screen.AllScreens.Length - 1); + + projector.displayIndex = Screen.AllScreens.Length - 1; + } + + var bounds = Screen.AllScreens[projector.displayIndex].Bounds; StartPosition = FormStartPosition.Manual; Location = new System.Drawing.Point(bounds.X, bounds.Y); Size = new Size(bounds.Width, bounds.Height); diff --git a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.cs b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.cs index 71354cd..cb5e62f 100644 --- a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.cs +++ b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.cs @@ -8,6 +8,8 @@ using System.Windows.Forms; using Microsoft.Kinect; using System.Runtime.InteropServices; +using System.Diagnostics; +using System.Linq; namespace RoomAliveToolkit { @@ -37,9 +39,6 @@ public ProjectionMappingSample(string[] args) // shaders depthAndColorShader = new DepthAndColorShader(device); projectiveTexturingShader = new ProjectiveTexturingShader(device); - passThroughShader = new PassThrough(device, userViewTextureWidth, userViewTextureHeight); - radialWobbleShader = new RadialWobble(device, userViewTextureWidth, userViewTextureHeight); - meshShader = new MeshShader(device); fromUIntPS = new FromUIntPS(device, Kinect2Calibration.depthImageWidth, Kinect2Calibration.depthImageHeight); bilateralFilter = new BilateralFilter(device, Kinect2Calibration.depthImageWidth, Kinect2Calibration.depthImageHeight); @@ -47,28 +46,6 @@ public ProjectionMappingSample(string[] args) foreach (var camera in ensemble.cameras) cameraDeviceResources[camera] = new CameraDeviceResource(device, camera, renderLock, directory); - // one user view - // user view render target, depth buffer, viewport for user view - var userViewTextureDesc = new Texture2DDescription() - { - Width = userViewTextureWidth, - Height = userViewTextureHeight, - MipLevels = 1, // revisit this; we may benefit from mipmapping? - ArraySize = 1, - Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Default, - BindFlags = BindFlags.RenderTarget | BindFlags.ShaderResource, - CpuAccessFlags = CpuAccessFlags.None, - }; - var userViewRenderTarget = new Texture2D(device, userViewTextureDesc); - userViewRenderTargetView = new RenderTargetView(device, userViewRenderTarget); - userViewSRV = new ShaderResourceView(device, userViewRenderTarget); - - var filteredUserViewRenderTarget = new Texture2D(device, userViewTextureDesc); - filteredUserViewRenderTargetView = new RenderTargetView(device, filteredUserViewRenderTarget); - filteredUserViewSRV = new ShaderResourceView(device, filteredUserViewRenderTarget); - // user view depth buffer var userViewDpethBufferDesc = new Texture2DDescription() { @@ -85,37 +62,6 @@ public ProjectionMappingSample(string[] args) var userViewDepthStencil = new Texture2D(device, userViewDpethBufferDesc); userViewDepthStencilView = new DepthStencilView(device, userViewDepthStencil); - // user view viewport - userViewViewport = new Viewport(0, 0, userViewTextureWidth, userViewTextureHeight, 0f, 1f); - - // desktop region texture - - if (desktopDuplicationEnabled) - { - windowPtr = FindWindow(null, "Untitled - Notepad"); - - // maybe use FindWindowEx to find child window - - var rect = new RECT(); - GetWindowRect(windowPtr, out rect); - var desktopTextureDesc = new Texture2DDescription() - { - Width = (rect.Right + rightNudge) - (rect.Left + leftNudge), - Height = (rect.Bottom + bottomNudge) - (rect.Top + topNudge), - MipLevels = 1, // revisit this; we may benefit from mipmapping? - ArraySize = 1, - Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Default, - BindFlags = BindFlags.ShaderResource, - CpuAccessFlags = CpuAccessFlags.None, - }; - desktopTexture = new Texture2D(device, desktopTextureDesc); - desktopTextureSRV = new ShaderResourceView(device, desktopTexture); - } - - - // create a form for each projector foreach (var projector in ensemble.projectors) { @@ -126,31 +72,7 @@ public ProjectionMappingSample(string[] args) projectorForms.Add(form); } - // example 3d object - var mesh = Mesh.FromOBJFile("Content/FloorPlan.obj"); - meshDeviceResources = new MeshDeviceResources(device, imagingFactory, mesh); - - // desktop duplication - var output = new Output1(factory.Adapters[0].Outputs[0].NativePointer); // TODO: change adapter and output number - outputDuplication = output.DuplicateOutput(device); - - - userViewForm = new MainForm(factory, device, renderLock); - userViewForm.Show(); - - - userViewForm.videoPanel1.MouseClick += videoPanel1_MouseClick; - - // connect to local camera to acquire head position - if (localHeadTrackingEnabled) - { - localKinectSensor = KinectSensor.GetDefault(); - bodyFrameReader = localKinectSensor.BodyFrameSource.OpenReader(); - localKinectSensor.Open(); - Console.WriteLine("connected to local camera"); - new System.Threading.Thread(LocalBodyLoop).Start(); - } - + clock.Start(); if (liveDepthEnabled) { @@ -158,19 +80,9 @@ public ProjectionMappingSample(string[] args) cameraDeviceResource.StartLive(); } - new System.Threading.Thread(RenderLoop).Start(); } - void videoPanel1_MouseClick(object sender, MouseEventArgs e) - { - alpha = 0; - } - - - OutputDuplication outputDuplication; - - const int userViewTextureWidth = 2000; const int userViewTextureHeight = 1000; List projectorForms = new List(); @@ -178,83 +90,39 @@ void videoPanel1_MouseClick(object sender, MouseEventArgs e) ProjectiveTexturingShader projectiveTexturingShader; Dictionary cameraDeviceResources = new Dictionary(); Object renderLock = new Object(); - RenderTargetView userViewRenderTargetView, filteredUserViewRenderTargetView; DepthStencilView userViewDepthStencilView; - ShaderResourceView userViewSRV, filteredUserViewSRV, desktopTextureSRV; - Viewport userViewViewport; SharpDX.Direct3D11.Device device; ProjectorCameraEnsemble ensemble; - Form1 userViewForm; - MeshShader meshShader; - MeshDeviceResources meshDeviceResources; - PassThrough passThroughShader; - RadialWobble radialWobbleShader; + FromUIntPS fromUIntPS; BilateralFilter bilateralFilter; System.Diagnostics.Stopwatch stopwatch = new System.Diagnostics.Stopwatch(); PointLight pointLight = new PointLight(); - Texture2D desktopTexture; - - IntPtr windowPtr; - const int leftNudge = 9; - const int topNudge = -10; - const int rightNudge = 0; - const int bottomNudge = 60; - SharpDX.WIC.ImagingFactory2 imagingFactory = new ImagingFactory2(); - float alpha = 1; // TODO: make so these can be changed live, put in menu - bool threeDObjectEnabled = Properties.Settings.Default.ThreeDObjectEnabled; - bool wobbleEffectEnabled = Properties.Settings.Default.WobbleEffectEnabled; - bool localHeadTrackingEnabled = Properties.Settings.Default.LocalHeadTrackingEnabled; bool liveDepthEnabled = Properties.Settings.Default.LiveDepthEnabled; bool fullScreenEnabled = Properties.Settings.Default.FullScreenEnabled; - bool desktopDuplicationEnabled = Properties.Settings.Default.DesktopDuplicationEnabled; + + Stopwatch clock = new Stopwatch(); void RenderLoop() { - - while (true) + while (true) { lock (renderLock) { var deviceContext = device.ImmediateContext; // render user view - deviceContext.ClearRenderTargetView(userViewRenderTargetView, Color4.Black); deviceContext.ClearDepthStencilView(userViewDepthStencilView, DepthStencilClearFlags.Depth, 1, 0); SharpDX.Vector3 headPosition = new SharpDX.Vector3(0f, 1.1f, -1.4f); // may need to change this default - if (localHeadTrackingEnabled) - { - float distanceSquared = 0; - lock (headCameraSpacePointLock) - { - headPosition = new SharpDX.Vector3(headCameraSpacePoint.X, headCameraSpacePoint.Y, headCameraSpacePoint.Z); - - float dx = handLeftCameraSpacePoint.X - handRightCameraSpacePoint.X; - float dy = handLeftCameraSpacePoint.Y - handRightCameraSpacePoint.Y; - float dz = handLeftCameraSpacePoint.Z - handRightCameraSpacePoint.Z; - distanceSquared = dx * dx + dy * dy + dz * dz; - } - var transform = SharpDX.Matrix.RotationY((float)Math.PI) * SharpDX.Matrix.Translation(-0.25f, 0.45f, 0); - headPosition = SharpDX.Vector3.TransformCoordinate(headPosition, transform); - - if (trackingValid && (distanceSquared < 0.02f) && (alpha > 1)) - alpha = 0; - //Console.WriteLine(distanceSquared); - } - var userView = GraphicsTransforms.LookAt(headPosition, headPosition + SharpDX.Vector3.UnitZ, SharpDX.Vector3.UnitY); userView.Transpose(); - - //Console.WriteLine("headPosition = " + headPosition); - - float aspect = (float)userViewTextureWidth / (float)userViewTextureHeight; var userProjection = GraphicsTransforms.PerspectiveFov(55.0f / 180.0f * (float)Math.PI, aspect, 0.001f, 1000.0f); userProjection.Transpose(); @@ -266,101 +134,12 @@ void RenderLoop() if (cameraDeviceResource.depthImageChanged) { fromUIntPS.Render(deviceContext, cameraDeviceResource.depthImageTextureRV, cameraDeviceResource.floatDepthImageRenderTargetView); - for (int i = 0; i < 1; i++) - { - bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.floatDepthImageRenderTargetView2); - bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV2, cameraDeviceResource.floatDepthImageRenderTargetView); - } + bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.floatDepthImageRenderTargetView2); + bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV2, cameraDeviceResource.floatDepthImageRenderTargetView); cameraDeviceResource.depthImageChanged = false; } } - // wobble effect - if (wobbleEffectEnabled) - foreach (var camera in ensemble.cameras) - { - var cameraDeviceResource = cameraDeviceResources[camera]; - - var world = new SharpDX.Matrix(); - for (int i = 0; i < 4; i++) - for (int j = 0; j < 4; j++) - world[i, j] = (float)camera.pose[i, j]; - world.Transpose(); - - // view and projection matrix are post-multiply - var userWorldViewProjection = world * userView * userProjection; - - depthAndColorShader.SetConstants(deviceContext, camera.calibration, userWorldViewProjection); - depthAndColorShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.colorImageTextureRV, cameraDeviceResource.vertexBuffer, userViewRenderTargetView, userViewDepthStencilView, userViewViewport); - } - - // 3d object - if (threeDObjectEnabled) - { - var world = SharpDX.Matrix.Scaling(1.0f) * SharpDX.Matrix.RotationY(90.0f / 180.0f * (float)Math.PI) * - SharpDX.Matrix.RotationX(-40.0f / 180.0f * (float)Math.PI) * SharpDX.Matrix.Translation(0, 0.7f, 0.0f); - - var pointLight = new PointLight(); - pointLight.position = new Vector3(0, 2, 0); - pointLight.Ia = new Vector3(0.1f, 0.1f, 0.1f); - meshShader.SetVertexShaderConstants(deviceContext, world, userView * userProjection, pointLight.position); - meshShader.Render(deviceContext, meshDeviceResources, pointLight, userViewRenderTargetView, userViewDepthStencilView, userViewViewport); - } - - // wobble effect - if (wobbleEffectEnabled) - { - alpha += 0.05f; - if (alpha > 1) - radialWobbleShader.SetConstants(deviceContext, 0); - else - radialWobbleShader.SetConstants(deviceContext, alpha); - radialWobbleShader.Render(deviceContext, userViewSRV, filteredUserViewRenderTargetView); - } - - - // desktop duplication - if (desktopDuplicationEnabled) - { - var outputDuplicateFrameInformation = default(OutputDuplicateFrameInformation); - SharpDX.DXGI.Resource resource = null; - outputDuplication.AcquireNextFrame(1000, out outputDuplicateFrameInformation, out resource); - var texture = resource.QueryInterface(); - - var rect = new RECT(); - GetWindowRect(windowPtr, out rect); - - var sourceRegion = new ResourceRegion() - { - Left = rect.Left + leftNudge, - Right = rect.Right + rightNudge, - Top = rect.Top + topNudge, - Bottom = rect.Bottom + bottomNudge, - Front = 0, - Back = 1, - }; - deviceContext.CopySubresourceRegion(texture, 0, sourceRegion, desktopTexture, 0); - texture.Dispose(); - } - - - // render user view to seperate form - passThroughShader.viewport = new Viewport(0, 0, userViewForm.Width, userViewForm.Height); - // TODO: clean this up by simply using a pointer to the userViewSRV - if (threeDObjectEnabled) - { - passThroughShader.Render(deviceContext, userViewSRV, userViewForm.renderTargetView); - } - if (wobbleEffectEnabled) - { - passThroughShader.Render(deviceContext, filteredUserViewSRV, userViewForm.renderTargetView); - } - if (desktopDuplicationEnabled) - { - passThroughShader.Render(deviceContext, desktopTextureSRV, userViewForm.renderTargetView); - } - userViewForm.swapChain.Present(0, PresentFlags.None); - // projection puts x and y in [-1,1]; adjust to obtain texture coordinates [0,1] // TODO: put this in SetContants? userProjection[0, 0] /= 2; @@ -378,6 +157,7 @@ void RenderLoop() { var cameraDeviceResource = cameraDeviceResources[camera]; + // Copy camera pose var world = new SharpDX.Matrix(); for (int i = 0; i < 4; i++) for (int j = 0; j < 4; j++) @@ -387,421 +167,19 @@ void RenderLoop() var projectorWorldViewProjection = world * form.view * form.projection; var userWorldViewProjection = world * userView * userProjection; - projectiveTexturingShader.SetConstants(deviceContext, userWorldViewProjection, projectorWorldViewProjection); - - // TODO: clean this up by simply using a pointer to the userViewSRV - if (wobbleEffectEnabled) - projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, filteredUserViewSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport); - if (threeDObjectEnabled) - projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, userViewSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport); - if (desktopDuplicationEnabled) - projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, desktopTextureSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport); + projectiveTexturingShader.SetConstants(deviceContext, userWorldViewProjection, projectorWorldViewProjection, clock.Elapsed); + projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport); } form.swapChain.Present(1, PresentFlags.None); } - - if (desktopDuplicationEnabled) - outputDuplication.ReleaseFrame(); - - - Console.WriteLine(stopwatch.ElapsedMilliseconds); + //Console.WriteLine(stopwatch.ElapsedMilliseconds); stopwatch.Restart(); } } } - KinectSensor localKinectSensor; - BodyFrameReader bodyFrameReader; - Body[] bodies = null; - CameraSpacePoint headCameraSpacePoint, handLeftCameraSpacePoint, handRightCameraSpacePoint; - Object headCameraSpacePointLock = new Object(); - bool trackingValid = false; - - void LocalBodyLoop() - { - while (true) - { - // find closest tracked head - var bodyFrame = bodyFrameReader.AcquireLatestFrame(); - - if (bodyFrame != null) - { - if (bodies == null) - bodies = new Body[bodyFrame.BodyCount]; - bodyFrame.GetAndRefreshBodyData(bodies); - - bool foundTrackedBody = false; - float distanceToNearest = float.MaxValue; - var nearestHeadCameraSpacePoint = new CameraSpacePoint(); - var nearestHandRightCameraSpacePoint = new CameraSpacePoint(); - var nearestHandLeftCameraSpacePoint = new CameraSpacePoint(); - - - foreach (var body in bodies) - if (body.IsTracked) - { - var cameraSpacePoint = body.Joints[JointType.Head].Position; - if (cameraSpacePoint.Z < distanceToNearest) - { - distanceToNearest = cameraSpacePoint.Z; - nearestHeadCameraSpacePoint = cameraSpacePoint; - nearestHandLeftCameraSpacePoint = body.Joints[JointType.HandLeft].Position; - nearestHandRightCameraSpacePoint = body.Joints[JointType.HandRight].Position; - foundTrackedBody = true; - } - } - - lock (headCameraSpacePointLock) - { - if (foundTrackedBody) - { - headCameraSpacePoint = nearestHeadCameraSpacePoint; - handLeftCameraSpacePoint = nearestHandLeftCameraSpacePoint; - handRightCameraSpacePoint = nearestHandRightCameraSpacePoint; - - trackingValid = true; - //Console.WriteLine("{0} {1} {2}", headCameraSpacePoint.X, headCameraSpacePoint.Y, headCameraSpacePoint.Z); - } - else - { - headCameraSpacePoint.X = 0f; - headCameraSpacePoint.Y = 0.3f; - headCameraSpacePoint.Z = 1.5f; - - trackingValid = false; - } - } - - bodyFrame.Dispose(); - } - else - System.Threading.Thread.Sleep(5); - } - } - - - - - class CameraDeviceResource : IDisposable - { - // encapsulates d3d resources for a camera - public CameraDeviceResource(SharpDX.Direct3D11.Device device, ProjectorCameraEnsemble.Camera camera, Object renderLock, string directory) - { - this.device = device; - this.camera = camera; - this.renderLock = renderLock; - - // Kinect depth image - var depthImageTextureDesc = new Texture2DDescription() - { - Width = Kinect2Calibration.depthImageWidth, - Height = Kinect2Calibration.depthImageHeight, - MipLevels = 1, - ArraySize = 1, - Format = SharpDX.DXGI.Format.R16_UInt, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Dynamic, - BindFlags = BindFlags.ShaderResource, - CpuAccessFlags = CpuAccessFlags.Write, - }; - depthImageTexture = new Texture2D(device, depthImageTextureDesc); - depthImageTextureRV = new ShaderResourceView(device, depthImageTexture); - - var floatDepthImageTextureDesc = new Texture2DDescription() - { - Width = Kinect2Calibration.depthImageWidth, - Height = Kinect2Calibration.depthImageHeight, - MipLevels = 1, - ArraySize = 1, - Format = SharpDX.DXGI.Format.R32_Float, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Default, - BindFlags = BindFlags.RenderTarget | BindFlags.ShaderResource, - CpuAccessFlags = CpuAccessFlags.None, - }; - - floatDepthImageTexture = new Texture2D(device, floatDepthImageTextureDesc); - floatDepthImageRV = new ShaderResourceView(device, floatDepthImageTexture); - floatDepthImageRenderTargetView = new RenderTargetView(device, floatDepthImageTexture); - - floatDepthImageTexture2 = new Texture2D(device, floatDepthImageTextureDesc); - floatDepthImageRV2 = new ShaderResourceView(device, floatDepthImageTexture2); - floatDepthImageRenderTargetView2 = new RenderTargetView(device, floatDepthImageTexture2); - - // Kinect color image - var colorImageStagingTextureDesc = new Texture2DDescription() - { - Width = Kinect2Calibration.colorImageWidth, - Height = Kinect2Calibration.colorImageHeight, - MipLevels = 1, - ArraySize = 1, - Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Dynamic, - BindFlags = BindFlags.ShaderResource, - CpuAccessFlags = CpuAccessFlags.Write - }; - colorImageStagingTexture = new Texture2D(device, colorImageStagingTextureDesc); - - var colorImageTextureDesc = new Texture2DDescription() - { - Width = Kinect2Calibration.colorImageWidth, - Height = Kinect2Calibration.colorImageHeight, - MipLevels = 0, - ArraySize = 1, - Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Default, - BindFlags = BindFlags.ShaderResource | BindFlags.RenderTarget, - CpuAccessFlags = CpuAccessFlags.None, - OptionFlags = ResourceOptionFlags.GenerateMipMaps - }; - colorImageTexture = new Texture2D(device, colorImageTextureDesc); - colorImageTextureRV = new ShaderResourceView(device, colorImageTexture); - - // vertex buffer - var table = camera.calibration.ComputeDepthFrameToCameraSpaceTable(); - int numVertices = 6 * (Kinect2Calibration.depthImageWidth - 1) * (Kinect2Calibration.depthImageHeight - 1); - var vertices = new VertexPosition[numVertices]; - - Int3[] quadOffsets = new Int3[] - { - new Int3(0, 0, 0), - new Int3(1, 0, 0), - new Int3(0, 1, 0), - new Int3(1, 0, 0), - new Int3(1, 1, 0), - new Int3(0, 1, 0), - }; - - int vertexIndex = 0; - for (int y = 0; y < Kinect2Calibration.depthImageHeight - 1; y++) - for (int x = 0; x < Kinect2Calibration.depthImageWidth - 1; x++) - for (int i = 0; i < 6; i++) - { - int vertexX = x + quadOffsets[i].X; - int vertexY = y + quadOffsets[i].Y; - - var point = table[Kinect2Calibration.depthImageWidth * vertexY + vertexX]; - - var vertex = new VertexPosition(); - vertex.position = new SharpDX.Vector4(point.X, point.Y, vertexX, vertexY); - vertices[vertexIndex++] = vertex; - } - - var stream = new DataStream(numVertices * VertexPosition.SizeInBytes, true, true); - stream.WriteRange(vertices); - stream.Position = 0; - - var vertexBufferDesc = new BufferDescription() - { - BindFlags = BindFlags.VertexBuffer, - CpuAccessFlags = CpuAccessFlags.None, - Usage = ResourceUsage.Default, - SizeInBytes = numVertices * VertexPosition.SizeInBytes, - }; - vertexBuffer = new SharpDX.Direct3D11.Buffer(device, stream, vertexBufferDesc); - - vertexBufferBinding = new VertexBufferBinding(vertexBuffer, VertexPosition.SizeInBytes, 0); - - stream.Dispose(); - - var colorImage = new RoomAliveToolkit.ARGBImage(Kinect2Calibration.colorImageWidth, Kinect2Calibration.colorImageHeight); - ProjectorCameraEnsemble.LoadFromTiff(imagingFactory, colorImage, directory + "/camera" + camera.name + "/color.tiff"); - - var depthImage = new RoomAliveToolkit.ShortImage(Kinect2Calibration.depthImageWidth, Kinect2Calibration.depthImageHeight); - ProjectorCameraEnsemble.LoadFromTiff(imagingFactory, depthImage, directory + "/camera" + camera.name + "/mean.tiff"); - - lock (renderLock) // necessary? - { - UpdateColorImage(device.ImmediateContext, colorImage.DataIntPtr); - UpdateDepthImage(device.ImmediateContext, depthImage.DataIntPtr); - } - - colorImage.Dispose(); - depthImage.Dispose(); - - - - - } - - struct VertexPosition - { - public SharpDX.Vector4 position; - static public int SizeInBytes { get { return 4 * 4; } } - } - - public void Dispose() - { - depthImageTexture.Dispose(); - depthImageTextureRV.Dispose(); - colorImageTexture.Dispose(); - colorImageTextureRV.Dispose(); - colorImageStagingTexture.Dispose(); - vertexBuffer.Dispose(); - } - - SharpDX.Direct3D11.Device device; - public Texture2D depthImageTexture, floatDepthImageTexture, floatDepthImageTexture2; - public ShaderResourceView depthImageTextureRV, floatDepthImageRV, floatDepthImageRV2; - public RenderTargetView floatDepthImageRenderTargetView, floatDepthImageRenderTargetView2; - public Texture2D colorImageTexture; - public ShaderResourceView colorImageTextureRV; - public Texture2D colorImageStagingTexture; - public SharpDX.Direct3D11.Buffer vertexBuffer; - VertexBufferBinding vertexBufferBinding; - ProjectorCameraEnsemble.Camera camera; - public bool renderEnabled = true; - - public void UpdateDepthImage(DeviceContext deviceContext, IntPtr depthImage) - { - DataStream dataStream; - deviceContext.MapSubresource(depthImageTexture, 0, - MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); - dataStream.WriteRange(depthImage, Kinect2Calibration.depthImageWidth * Kinect2Calibration.depthImageHeight * 2); - deviceContext.UnmapSubresource(depthImageTexture, 0); - } - - public void UpdateDepthImage(DeviceContext deviceContext, byte[] depthImage) - { - DataStream dataStream; - deviceContext.MapSubresource(depthImageTexture, 0, - MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); - dataStream.WriteRange(depthImage, 0, Kinect2Calibration.depthImageWidth * Kinect2Calibration.depthImageHeight * 2); - deviceContext.UnmapSubresource(depthImageTexture, 0); - } - - public void UpdateColorImage(DeviceContext deviceContext, IntPtr colorImage) - { - DataStream dataStream; - deviceContext.MapSubresource(colorImageStagingTexture, 0, - MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); - dataStream.WriteRange(colorImage, Kinect2Calibration.colorImageWidth * Kinect2Calibration.colorImageHeight * 4); - deviceContext.UnmapSubresource(colorImageStagingTexture, 0); - - var resourceRegion = new ResourceRegion() - { - Left = 0, - Top = 0, - Right = Kinect2Calibration.colorImageWidth, - Bottom = Kinect2Calibration.colorImageHeight, - Front = 0, - Back = 1, - }; - deviceContext.CopySubresourceRegion(colorImageStagingTexture, 0, resourceRegion, colorImageTexture, 0); - deviceContext.GenerateMips(colorImageTextureRV); - } - - public void UpdateColorImage(DeviceContext deviceContext, byte[] colorImage) - { - DataStream dataStream; - deviceContext.MapSubresource(colorImageStagingTexture, 0, - MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); - dataStream.WriteRange(colorImage, 0, Kinect2Calibration.colorImageWidth * Kinect2Calibration.colorImageHeight * 4); - deviceContext.UnmapSubresource(colorImageStagingTexture, 0); - - var resourceRegion = new ResourceRegion() - { - Left = 0, - Top = 0, - Right = Kinect2Calibration.colorImageWidth, - Bottom = Kinect2Calibration.colorImageHeight, - Front = 0, - Back = 1, - }; - deviceContext.CopySubresourceRegion(colorImageStagingTexture, 0, resourceRegion, colorImageTexture, 0); - deviceContext.GenerateMips(colorImageTextureRV); - } - - public void Render(DeviceContext deviceContext) - { - deviceContext.InputAssembler.SetVertexBuffers(0, vertexBufferBinding); - deviceContext.VertexShader.SetShaderResource(0, depthImageTextureRV); - deviceContext.PixelShader.SetShaderResource(0, colorImageTextureRV); - deviceContext.Draw((Kinect2Calibration.depthImageWidth - 1) * (Kinect2Calibration.depthImageHeight - 1) * 6, 0); - } - - bool live = false; - - public void StartLive() - { - live = true; - //new System.Threading.Thread(ColorCameraLoop).Start(); - new System.Threading.Thread(DepthCameraLoop).Start(); - } - - public void StopLive() - { - live = false; - } - - - Object renderLock; - public bool depthImageChanged = true; - - //byte[] colorData = new byte[4 * Kinect2.Kinect2Calibration.colorImageWidth * Kinect2.Kinect2Calibration.colorImageHeight]; - byte[] nextColorData = new byte[4 * RoomAliveToolkit.Kinect2Calibration.colorImageWidth * RoomAliveToolkit.Kinect2Calibration.colorImageHeight]; - SharpDX.WIC.ImagingFactory2 imagingFactory = new SharpDX.WIC.ImagingFactory2(); - void ColorCameraLoop() - { - while (true) - { - var encodedColorData = camera.Client.LatestJPEGImage(); - - // decode JPEG - var memoryStream = new MemoryStream(encodedColorData); - var stream = new WICStream(imagingFactory, memoryStream); - // decodes to 24 bit BGR - var decoder = new SharpDX.WIC.BitmapDecoder(imagingFactory, stream, SharpDX.WIC.DecodeOptions.CacheOnLoad); - var bitmapFrameDecode = decoder.GetFrame(0); - - // convert to 32 bpp - var formatConverter = new FormatConverter(imagingFactory); - formatConverter.Initialize(bitmapFrameDecode, SharpDX.WIC.PixelFormat.Format32bppBGR); - formatConverter.CopyPixels(nextColorData, Kinect2Calibration.colorImageWidth * 4); // TODO: consider copying directly to texture native memory - //lock (colorData) - // Swap(ref colorData, ref nextColorData); - lock (renderLock) // necessary? - { - UpdateColorImage(device.ImmediateContext, nextColorData); - } - memoryStream.Close(); - memoryStream.Dispose(); - stream.Dispose(); - decoder.Dispose(); - formatConverter.Dispose(); - bitmapFrameDecode.Dispose(); - } - } - - //byte[] depthData = new byte[2 * Kinect2.Kinect2Calibration.depthImageWidth * Kinect2.Kinect2Calibration.depthImageHeight]; - byte[] nextDepthData; - void DepthCameraLoop() - { - while (true) - { - nextDepthData = camera.Client.LatestDepthImage(); - //lock (remoteDepthData) - // Swap(ref remoteDepthData, ref nextRemoteDepthData); - lock (renderLock) - { - depthImageChanged = true; - UpdateDepthImage(device.ImmediateContext, nextDepthData); - } - } - } - - static void Swap(ref T first, ref T second) - { - T temp = first; - first = second; - second = temp; - } - } - [DllImport("user32.dll", SetLastError = true)] static extern IntPtr FindWindow(string lpClassName, string lpWindowName); diff --git a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj index d10b8ab..8d99a73 100644 --- a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj +++ b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj @@ -77,6 +77,7 @@ + diff --git a/ProCamCalibration/ProjectionMappingSample/ProjectiveTexturingShader.cs b/ProCamCalibration/ProjectionMappingSample/ProjectiveTexturingShader.cs index 5452b52..3ea5493 100644 --- a/ProCamCalibration/ProjectionMappingSample/ProjectiveTexturingShader.cs +++ b/ProCamCalibration/ProjectionMappingSample/ProjectiveTexturingShader.cs @@ -2,6 +2,7 @@ using SharpDX.D3DCompiler; using SharpDX.Direct3D11; using SharpDX.DXGI; +using System; using System.IO; using System.Runtime.InteropServices; using Device = SharpDX.Direct3D11.Device; @@ -76,24 +77,33 @@ public ProjectiveTexturingShader(Device device) [StructLayout(LayoutKind.Explicit, Size = Constants.size)] public unsafe struct Constants { - public const int size = 128; + public const int size = 128 + 16; [FieldOffset(0)] public fixed float userWorldViewProjection[16]; [FieldOffset(64)] public fixed float projectorWorldViewProjection[16]; + [FieldOffset(128)] + public float globalTime; }; - public unsafe void SetConstants(DeviceContext deviceContext, SharpDX.Matrix userWorldViewProjection, SharpDX.Matrix projectorWorldViewProjection) + public unsafe void SetConstants(DeviceContext deviceContext, + SharpDX.Matrix userWorldViewProjection, + SharpDX.Matrix projectorWorldViewProjection, + TimeSpan elapsedTime) { Constants constants = new Constants(); for (int i = 0, col = 0; col < 4; col++) + { for (int row = 0; row < 4; row++) { constants.userWorldViewProjection[i] = userWorldViewProjection[row, col]; constants.projectorWorldViewProjection[i] = projectorWorldViewProjection[row, col]; i++; } + } + + constants.globalTime = (float)elapsedTime.TotalSeconds; DataStream dataStream; deviceContext.MapSubresource(constantBuffer, MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); @@ -101,7 +111,7 @@ public unsafe void SetConstants(DeviceContext deviceContext, SharpDX.Matrix user deviceContext.UnmapSubresource(constantBuffer, 0); } - public void Render(DeviceContext deviceContext, ShaderResourceView depthImageTextureRV, ShaderResourceView colorImageTextureRV, SharpDX.Direct3D11.Buffer vertexBuffer, RenderTargetView renderTargetView, DepthStencilView depthStencilView, Viewport viewport) + public void Render(DeviceContext deviceContext, ShaderResourceView depthImageTextureRV, SharpDX.Direct3D11.Buffer vertexBuffer, RenderTargetView renderTargetView, DepthStencilView depthStencilView, Viewport viewport) { deviceContext.InputAssembler.InputLayout = vertexInputLayout; deviceContext.InputAssembler.PrimitiveTopology = SharpDX.Direct3D.PrimitiveTopology.TriangleList; @@ -115,8 +125,7 @@ public void Render(DeviceContext deviceContext, ShaderResourceView depthImageTex deviceContext.VertexShader.SetConstantBuffer(0, constantBuffer); deviceContext.GeometryShader.Set(geometryShader); deviceContext.PixelShader.Set(pixelShader); - deviceContext.PixelShader.SetShaderResource(0, colorImageTextureRV); - deviceContext.PixelShader.SetSampler(0, colorSamplerState); + deviceContext.PixelShader.SetConstantBuffer(0, constantBuffer); deviceContext.Draw((Kinect2Calibration.depthImageWidth - 1) * (Kinect2Calibration.depthImageHeight - 1) * 6, 0); deviceContext.VertexShader.SetShaderResource(0, null); // to avoid warnings when these are later set as render targets diff --git a/ProCamCalibration/ProjectionMappingSample/Properties/Settings.Designer.cs b/ProCamCalibration/ProjectionMappingSample/Properties/Settings.Designer.cs index 0f4bcc0..c65978b 100644 --- a/ProCamCalibration/ProjectionMappingSample/Properties/Settings.Designer.cs +++ b/ProCamCalibration/ProjectionMappingSample/Properties/Settings.Designer.cs @@ -26,42 +26,6 @@ public static Settings Default { [global::System.Configuration.UserScopedSettingAttribute()] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Configuration.DefaultSettingValueAttribute("True")] - public bool ThreeDObjectEnabled { - get { - return ((bool)(this["ThreeDObjectEnabled"])); - } - set { - this["ThreeDObjectEnabled"] = value; - } - } - - [global::System.Configuration.UserScopedSettingAttribute()] - [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("False")] - public bool WobbleEffectEnabled { - get { - return ((bool)(this["WobbleEffectEnabled"])); - } - set { - this["WobbleEffectEnabled"] = value; - } - } - - [global::System.Configuration.UserScopedSettingAttribute()] - [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("False")] - public bool LocalHeadTrackingEnabled { - get { - return ((bool)(this["LocalHeadTrackingEnabled"])); - } - set { - this["LocalHeadTrackingEnabled"] = value; - } - } - - [global::System.Configuration.UserScopedSettingAttribute()] - [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("False")] public bool LiveDepthEnabled { get { return ((bool)(this["LiveDepthEnabled"])); @@ -73,7 +37,7 @@ public bool LiveDepthEnabled { [global::System.Configuration.UserScopedSettingAttribute()] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("False")] + [global::System.Configuration.DefaultSettingValueAttribute("True")] public bool FullScreenEnabled { get { return ((bool)(this["FullScreenEnabled"])); @@ -82,17 +46,5 @@ public bool FullScreenEnabled { this["FullScreenEnabled"] = value; } } - - [global::System.Configuration.UserScopedSettingAttribute()] - [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("False")] - public bool DesktopDuplicationEnabled { - get { - return ((bool)(this["DesktopDuplicationEnabled"])); - } - set { - this["DesktopDuplicationEnabled"] = value; - } - } } } diff --git a/ProCamCalibration/ProjectionMappingSample/Properties/Settings.settings b/ProCamCalibration/ProjectionMappingSample/Properties/Settings.settings index 9715df8..54a9f91 100644 --- a/ProCamCalibration/ProjectionMappingSample/Properties/Settings.settings +++ b/ProCamCalibration/ProjectionMappingSample/Properties/Settings.settings @@ -2,23 +2,11 @@ - - True - - - False - - - False - - False + True - False - - - False + True \ No newline at end of file diff --git a/ProCamCalibration/Shaders/DepthAndColorGS.hlsl b/ProCamCalibration/Shaders/DepthAndColorGS.hlsl index 9ab029f..437a2f7 100644 --- a/ProCamCalibration/Shaders/DepthAndColorGS.hlsl +++ b/ProCamCalibration/Shaders/DepthAndColorGS.hlsl @@ -3,6 +3,7 @@ struct GSInput float4 pos : SV_POSITION; float2 tex : TEXCOORD0; float depth : MYSEMANTIC; + float4 worldPos : TEXCOORD1; }; struct GSOutput @@ -10,6 +11,7 @@ struct GSOutput float4 pos : SV_POSITION; float2 tex : TEXCOORD0; float depth : MYSEMANTIC; + float4 worldPos : TEXCOORD1; }; [maxvertexcount(3)] diff --git a/ProCamCalibration/Shaders/DepthAndColorPS.hlsl b/ProCamCalibration/Shaders/DepthAndColorPS.hlsl index 733e86b..79c4517 100644 --- a/ProCamCalibration/Shaders/DepthAndColorPS.hlsl +++ b/ProCamCalibration/Shaders/DepthAndColorPS.hlsl @@ -1,14 +1,112 @@ -Texture2D colorTexture : register(t0); -SamplerState colorSampler : register(s0); struct PSInput { float4 pos : SV_POSITION; float2 tex : TEXCOORD0; float depth : MYSEMANTIC; + float4 worldPos : TEXCOORD1; }; -float4 main(PSInput input) : SV_TARGET0 +cbuffer constants : register(b0) +{ + // world is depth camera's pose; view and projection are that of the user + // also includes adjustment to go from view volume to texture coordinates + matrix userWorldViewProjection; + // world is depth camera's pose; view and projection are that of the projector + matrix projectorWorldViewProjection; + + float globalTime; +} + + +// Based on shaders: +// Template - 3D https://www.shadertoy.com/view/ldfSWs +// Xor - Triangle Grid, https://www.shadertoy.com/view/4tSGWz + +#define pi 3.14159265358979 +#define size 0.5 +#define reciproce_sqrt3 0.57735026918962576450914878050196 +#define lineThickness 0.01 + +float planeDistance = 0.2; +float offset; + +float r(float n) +{ + return frac(abs(sin(n*55.753)*367.34)); +} + +float r(float2 n) +{ + return r(dot(n, float2(2.46, -1.21))); +} + +float3 smallTrianglesColor(float3 pos) +{ + float a = (radians(60.0)); + float zoom = 0.125; + float2 c = (pos.xy + float2(0.0, pos.z)) * float2(sin(a), 1.0);//scaled coordinates + c = ((c + float2(c.y, 0.0)*cos(a)) / zoom) + float2(floor((c.x - c.y*cos(a)) / zoom*4.0) / 4.0, 0.0);//Add rotations + float type = (r(floor(c*4.0))*0.2 + r(floor(c*2.0))*0.3 + r(floor(c))*0.5);//Randomize type + type += 0.3 * sin(globalTime*5.0*type); + + float l = min(min((1.0 - (2.0 * abs(frac((c.x - c.y)*4.0) - 0.5))), + (1.0 - (2.0 * abs(frac(c.y * 4.0) - 0.5)))), + (1.0 - (2.0 * abs(frac(c.x * 4.0) - 0.5)))); + l = smoothstep(0.06, 0.04, l); + + return lerp(type, l, 0.3); +} + +float3 largeTrianglesColor(float3 pos) +{ + float a = (radians(60.0)); + float zoom = 0.5; + float2 c = (pos.xy + float2(0.0, pos.z)) * float2(sin(a), 1.0);//scaled coordinates + c = ((c + float2(c.y, 0.0)*cos(a)) / zoom) + float2(floor((c.x - c.y*cos(a)) / zoom*4.0) / 4.0, 0.0);//Add rotations + + float l = min(min((1.0 - (2.0 * abs(frac((c.x - c.y)*4.0) - 0.5))), + (1.0 - (2.0 * abs(frac(c.y * 4.0) - 0.5)))), + (1.0 - (2.0 * abs(frac(c.x * 4.0) - 0.5)))); + l = smoothstep(0.03, 0.02, l); + + return lerp(0.01, l, 1.0); +} + +//--------------------------------------------------------------- +// Material +// +// Defines the material (colors, shading, pattern, texturing) of the model +// at every point based on its position and normal. In this case, it simply +// returns a constant yellow color. +//------------------------------------------------------------------------ +float3 doMaterial(in float3 pos, in float3 midPos) { - return colorTexture.Sample(colorSampler, input.tex); + float d = length(pos.xz - midPos.xz) + pos.y - midPos.y; + + float border = fmod(globalTime, 5.0); + + float3 c1 = largeTrianglesColor(pos); + float3 c2 = smallTrianglesColor(pos); + + // Small rim + float3 c = float3(1.0, 1.0, 1.0) * smoothstep(border - 0.2, border, d); + // Large Triangles to all + c += c1; + // Small triangle slightly after front + c += c2 * smoothstep(border - 0.4, border - 0.6, d); + // Cutoff front + c *= smoothstep(border, border - 0.05, d); + // Cutoff back + c *= smoothstep(border - 3.0, border - 0.5, d); + // fade out + c *= smoothstep(5.0, 4.0, border); + + return c * float3(0.2, 0.5, 1.0); } + + +float4 main(PSInput input) : SV_TARGET0 +{ + return float4(doMaterial(input.worldPos.xyz, float3(0.0, -1.0, 3.0)), 1.0); +} \ No newline at end of file diff --git a/ProCamCalibration/Shaders/DepthAndProjectiveTextureVS.hlsl b/ProCamCalibration/Shaders/DepthAndProjectiveTextureVS.hlsl index d35b4cf..72fcaf9 100644 --- a/ProCamCalibration/Shaders/DepthAndProjectiveTextureVS.hlsl +++ b/ProCamCalibration/Shaders/DepthAndProjectiveTextureVS.hlsl @@ -10,6 +10,7 @@ struct VSOutput float4 pos : SV_POSITION; float2 tex : TEXCOORD0; float depth : MYSEMANTIC; + float4 worldPos : TEXCOORD1; }; cbuffer constants : register(b0) @@ -19,6 +20,8 @@ cbuffer constants : register(b0) matrix userWorldViewProjection; // world is depth camera's pose; view and projection are that of the projector matrix projectorWorldViewProjection; + + float globalTime; } // Input vertices are setup so that xy are the entries of the table @@ -52,6 +55,7 @@ VSOutput main(VSInput input) output.pos = pos; output.tex = userViewTex.xy; output.depth = depth; // passed to geometry shader to cull invalid triangles + output.worldPos = depthCamera; return output; } From 21544a2be853e54d77846c52f259546ca6394044 Mon Sep 17 00:00:00 2001 From: Roland Smeenk Date: Tue, 2 Feb 2016 13:46:23 +0100 Subject: [PATCH 2/9] Reverted to original after placing these changes on separate branch --- .../ProjectionMappingSample/App.config | 32 +- .../DepthAndColorShader.cs | 4 - .../ProjectionMappingSample/Form1.cs | 11 +- .../ProjectionMappingSample.cs | 648 +----------------- .../ProjectionMappingSample.csproj | 1 + .../ProjectiveTexturingShader.cs | 19 +- .../Properties/Settings.Designer.cs | 50 +- .../Properties/Settings.settings | 16 +- .../Shaders/DepthAndColorGS.hlsl | 2 + .../Shaders/DepthAndColorPS.hlsl | 106 ++- .../Shaders/DepthAndProjectiveTextureVS.hlsl | 4 + 11 files changed, 159 insertions(+), 734 deletions(-) diff --git a/ProCamCalibration/ProjectionMappingSample/App.config b/ProCamCalibration/ProjectionMappingSample/App.config index 272bc19..2223e7c 100644 --- a/ProCamCalibration/ProjectionMappingSample/App.config +++ b/ProCamCalibration/ProjectionMappingSample/App.config @@ -8,26 +8,14 @@ - - - - True - - - False - - - False - - - False - - - False - - - False - - - + + + + True + + + True + + + \ No newline at end of file diff --git a/ProCamCalibration/ProjectionMappingSample/DepthAndColorShader.cs b/ProCamCalibration/ProjectionMappingSample/DepthAndColorShader.cs index 1a777aa..f13eb8e 100644 --- a/ProCamCalibration/ProjectionMappingSample/DepthAndColorShader.cs +++ b/ProCamCalibration/ProjectionMappingSample/DepthAndColorShader.cs @@ -293,12 +293,8 @@ public void Render(DeviceContext deviceContext, ShaderResourceView depthImageTex SamplerState colorSamplerState; SharpDX.Direct3D11.Buffer constantBuffer; BilateralFilter bilateralFilter; - FromUIntPS fromUIntPS; Texture2D filteredDepthImageTexture, filteredDepthImageTexture2; RenderTargetView filteredRenderTargetView, filteredRenderTargetView2; ShaderResourceView filteredDepthImageSRV, filteredDepthImageSRV2; } - - - } diff --git a/ProCamCalibration/ProjectionMappingSample/Form1.cs b/ProCamCalibration/ProjectionMappingSample/Form1.cs index 165b6bb..1a40a92 100644 --- a/ProCamCalibration/ProjectionMappingSample/Form1.cs +++ b/ProCamCalibration/ProjectionMappingSample/Form1.cs @@ -133,7 +133,16 @@ public bool FullScreen // switch to fullscreen ShowInTaskbar = false; FormBorderStyle = FormBorderStyle.None; - var bounds = Screen.AllScreens[projector.displayIndex].Bounds; // TODO: catch the case where the display is not available + + // Clamp to last screen if screen number is not available + if (projector.displayIndex >= Screen.AllScreens.Length) + { + Console.WriteLine("Projector '{0}' Clamped displayIndex '{1}' to '{2}'", projector.name, projector.displayIndex, Screen.AllScreens.Length - 1); + + projector.displayIndex = Screen.AllScreens.Length - 1; + } + + var bounds = Screen.AllScreens[projector.displayIndex].Bounds; StartPosition = FormStartPosition.Manual; Location = new System.Drawing.Point(bounds.X, bounds.Y); Size = new Size(bounds.Width, bounds.Height); diff --git a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.cs b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.cs index 71354cd..cb5e62f 100644 --- a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.cs +++ b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.cs @@ -8,6 +8,8 @@ using System.Windows.Forms; using Microsoft.Kinect; using System.Runtime.InteropServices; +using System.Diagnostics; +using System.Linq; namespace RoomAliveToolkit { @@ -37,9 +39,6 @@ public ProjectionMappingSample(string[] args) // shaders depthAndColorShader = new DepthAndColorShader(device); projectiveTexturingShader = new ProjectiveTexturingShader(device); - passThroughShader = new PassThrough(device, userViewTextureWidth, userViewTextureHeight); - radialWobbleShader = new RadialWobble(device, userViewTextureWidth, userViewTextureHeight); - meshShader = new MeshShader(device); fromUIntPS = new FromUIntPS(device, Kinect2Calibration.depthImageWidth, Kinect2Calibration.depthImageHeight); bilateralFilter = new BilateralFilter(device, Kinect2Calibration.depthImageWidth, Kinect2Calibration.depthImageHeight); @@ -47,28 +46,6 @@ public ProjectionMappingSample(string[] args) foreach (var camera in ensemble.cameras) cameraDeviceResources[camera] = new CameraDeviceResource(device, camera, renderLock, directory); - // one user view - // user view render target, depth buffer, viewport for user view - var userViewTextureDesc = new Texture2DDescription() - { - Width = userViewTextureWidth, - Height = userViewTextureHeight, - MipLevels = 1, // revisit this; we may benefit from mipmapping? - ArraySize = 1, - Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Default, - BindFlags = BindFlags.RenderTarget | BindFlags.ShaderResource, - CpuAccessFlags = CpuAccessFlags.None, - }; - var userViewRenderTarget = new Texture2D(device, userViewTextureDesc); - userViewRenderTargetView = new RenderTargetView(device, userViewRenderTarget); - userViewSRV = new ShaderResourceView(device, userViewRenderTarget); - - var filteredUserViewRenderTarget = new Texture2D(device, userViewTextureDesc); - filteredUserViewRenderTargetView = new RenderTargetView(device, filteredUserViewRenderTarget); - filteredUserViewSRV = new ShaderResourceView(device, filteredUserViewRenderTarget); - // user view depth buffer var userViewDpethBufferDesc = new Texture2DDescription() { @@ -85,37 +62,6 @@ public ProjectionMappingSample(string[] args) var userViewDepthStencil = new Texture2D(device, userViewDpethBufferDesc); userViewDepthStencilView = new DepthStencilView(device, userViewDepthStencil); - // user view viewport - userViewViewport = new Viewport(0, 0, userViewTextureWidth, userViewTextureHeight, 0f, 1f); - - // desktop region texture - - if (desktopDuplicationEnabled) - { - windowPtr = FindWindow(null, "Untitled - Notepad"); - - // maybe use FindWindowEx to find child window - - var rect = new RECT(); - GetWindowRect(windowPtr, out rect); - var desktopTextureDesc = new Texture2DDescription() - { - Width = (rect.Right + rightNudge) - (rect.Left + leftNudge), - Height = (rect.Bottom + bottomNudge) - (rect.Top + topNudge), - MipLevels = 1, // revisit this; we may benefit from mipmapping? - ArraySize = 1, - Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Default, - BindFlags = BindFlags.ShaderResource, - CpuAccessFlags = CpuAccessFlags.None, - }; - desktopTexture = new Texture2D(device, desktopTextureDesc); - desktopTextureSRV = new ShaderResourceView(device, desktopTexture); - } - - - // create a form for each projector foreach (var projector in ensemble.projectors) { @@ -126,31 +72,7 @@ public ProjectionMappingSample(string[] args) projectorForms.Add(form); } - // example 3d object - var mesh = Mesh.FromOBJFile("Content/FloorPlan.obj"); - meshDeviceResources = new MeshDeviceResources(device, imagingFactory, mesh); - - // desktop duplication - var output = new Output1(factory.Adapters[0].Outputs[0].NativePointer); // TODO: change adapter and output number - outputDuplication = output.DuplicateOutput(device); - - - userViewForm = new MainForm(factory, device, renderLock); - userViewForm.Show(); - - - userViewForm.videoPanel1.MouseClick += videoPanel1_MouseClick; - - // connect to local camera to acquire head position - if (localHeadTrackingEnabled) - { - localKinectSensor = KinectSensor.GetDefault(); - bodyFrameReader = localKinectSensor.BodyFrameSource.OpenReader(); - localKinectSensor.Open(); - Console.WriteLine("connected to local camera"); - new System.Threading.Thread(LocalBodyLoop).Start(); - } - + clock.Start(); if (liveDepthEnabled) { @@ -158,19 +80,9 @@ public ProjectionMappingSample(string[] args) cameraDeviceResource.StartLive(); } - new System.Threading.Thread(RenderLoop).Start(); } - void videoPanel1_MouseClick(object sender, MouseEventArgs e) - { - alpha = 0; - } - - - OutputDuplication outputDuplication; - - const int userViewTextureWidth = 2000; const int userViewTextureHeight = 1000; List projectorForms = new List(); @@ -178,83 +90,39 @@ void videoPanel1_MouseClick(object sender, MouseEventArgs e) ProjectiveTexturingShader projectiveTexturingShader; Dictionary cameraDeviceResources = new Dictionary(); Object renderLock = new Object(); - RenderTargetView userViewRenderTargetView, filteredUserViewRenderTargetView; DepthStencilView userViewDepthStencilView; - ShaderResourceView userViewSRV, filteredUserViewSRV, desktopTextureSRV; - Viewport userViewViewport; SharpDX.Direct3D11.Device device; ProjectorCameraEnsemble ensemble; - Form1 userViewForm; - MeshShader meshShader; - MeshDeviceResources meshDeviceResources; - PassThrough passThroughShader; - RadialWobble radialWobbleShader; + FromUIntPS fromUIntPS; BilateralFilter bilateralFilter; System.Diagnostics.Stopwatch stopwatch = new System.Diagnostics.Stopwatch(); PointLight pointLight = new PointLight(); - Texture2D desktopTexture; - - IntPtr windowPtr; - const int leftNudge = 9; - const int topNudge = -10; - const int rightNudge = 0; - const int bottomNudge = 60; - SharpDX.WIC.ImagingFactory2 imagingFactory = new ImagingFactory2(); - float alpha = 1; // TODO: make so these can be changed live, put in menu - bool threeDObjectEnabled = Properties.Settings.Default.ThreeDObjectEnabled; - bool wobbleEffectEnabled = Properties.Settings.Default.WobbleEffectEnabled; - bool localHeadTrackingEnabled = Properties.Settings.Default.LocalHeadTrackingEnabled; bool liveDepthEnabled = Properties.Settings.Default.LiveDepthEnabled; bool fullScreenEnabled = Properties.Settings.Default.FullScreenEnabled; - bool desktopDuplicationEnabled = Properties.Settings.Default.DesktopDuplicationEnabled; + + Stopwatch clock = new Stopwatch(); void RenderLoop() { - - while (true) + while (true) { lock (renderLock) { var deviceContext = device.ImmediateContext; // render user view - deviceContext.ClearRenderTargetView(userViewRenderTargetView, Color4.Black); deviceContext.ClearDepthStencilView(userViewDepthStencilView, DepthStencilClearFlags.Depth, 1, 0); SharpDX.Vector3 headPosition = new SharpDX.Vector3(0f, 1.1f, -1.4f); // may need to change this default - if (localHeadTrackingEnabled) - { - float distanceSquared = 0; - lock (headCameraSpacePointLock) - { - headPosition = new SharpDX.Vector3(headCameraSpacePoint.X, headCameraSpacePoint.Y, headCameraSpacePoint.Z); - - float dx = handLeftCameraSpacePoint.X - handRightCameraSpacePoint.X; - float dy = handLeftCameraSpacePoint.Y - handRightCameraSpacePoint.Y; - float dz = handLeftCameraSpacePoint.Z - handRightCameraSpacePoint.Z; - distanceSquared = dx * dx + dy * dy + dz * dz; - } - var transform = SharpDX.Matrix.RotationY((float)Math.PI) * SharpDX.Matrix.Translation(-0.25f, 0.45f, 0); - headPosition = SharpDX.Vector3.TransformCoordinate(headPosition, transform); - - if (trackingValid && (distanceSquared < 0.02f) && (alpha > 1)) - alpha = 0; - //Console.WriteLine(distanceSquared); - } - var userView = GraphicsTransforms.LookAt(headPosition, headPosition + SharpDX.Vector3.UnitZ, SharpDX.Vector3.UnitY); userView.Transpose(); - - //Console.WriteLine("headPosition = " + headPosition); - - float aspect = (float)userViewTextureWidth / (float)userViewTextureHeight; var userProjection = GraphicsTransforms.PerspectiveFov(55.0f / 180.0f * (float)Math.PI, aspect, 0.001f, 1000.0f); userProjection.Transpose(); @@ -266,101 +134,12 @@ void RenderLoop() if (cameraDeviceResource.depthImageChanged) { fromUIntPS.Render(deviceContext, cameraDeviceResource.depthImageTextureRV, cameraDeviceResource.floatDepthImageRenderTargetView); - for (int i = 0; i < 1; i++) - { - bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.floatDepthImageRenderTargetView2); - bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV2, cameraDeviceResource.floatDepthImageRenderTargetView); - } + bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.floatDepthImageRenderTargetView2); + bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV2, cameraDeviceResource.floatDepthImageRenderTargetView); cameraDeviceResource.depthImageChanged = false; } } - // wobble effect - if (wobbleEffectEnabled) - foreach (var camera in ensemble.cameras) - { - var cameraDeviceResource = cameraDeviceResources[camera]; - - var world = new SharpDX.Matrix(); - for (int i = 0; i < 4; i++) - for (int j = 0; j < 4; j++) - world[i, j] = (float)camera.pose[i, j]; - world.Transpose(); - - // view and projection matrix are post-multiply - var userWorldViewProjection = world * userView * userProjection; - - depthAndColorShader.SetConstants(deviceContext, camera.calibration, userWorldViewProjection); - depthAndColorShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.colorImageTextureRV, cameraDeviceResource.vertexBuffer, userViewRenderTargetView, userViewDepthStencilView, userViewViewport); - } - - // 3d object - if (threeDObjectEnabled) - { - var world = SharpDX.Matrix.Scaling(1.0f) * SharpDX.Matrix.RotationY(90.0f / 180.0f * (float)Math.PI) * - SharpDX.Matrix.RotationX(-40.0f / 180.0f * (float)Math.PI) * SharpDX.Matrix.Translation(0, 0.7f, 0.0f); - - var pointLight = new PointLight(); - pointLight.position = new Vector3(0, 2, 0); - pointLight.Ia = new Vector3(0.1f, 0.1f, 0.1f); - meshShader.SetVertexShaderConstants(deviceContext, world, userView * userProjection, pointLight.position); - meshShader.Render(deviceContext, meshDeviceResources, pointLight, userViewRenderTargetView, userViewDepthStencilView, userViewViewport); - } - - // wobble effect - if (wobbleEffectEnabled) - { - alpha += 0.05f; - if (alpha > 1) - radialWobbleShader.SetConstants(deviceContext, 0); - else - radialWobbleShader.SetConstants(deviceContext, alpha); - radialWobbleShader.Render(deviceContext, userViewSRV, filteredUserViewRenderTargetView); - } - - - // desktop duplication - if (desktopDuplicationEnabled) - { - var outputDuplicateFrameInformation = default(OutputDuplicateFrameInformation); - SharpDX.DXGI.Resource resource = null; - outputDuplication.AcquireNextFrame(1000, out outputDuplicateFrameInformation, out resource); - var texture = resource.QueryInterface(); - - var rect = new RECT(); - GetWindowRect(windowPtr, out rect); - - var sourceRegion = new ResourceRegion() - { - Left = rect.Left + leftNudge, - Right = rect.Right + rightNudge, - Top = rect.Top + topNudge, - Bottom = rect.Bottom + bottomNudge, - Front = 0, - Back = 1, - }; - deviceContext.CopySubresourceRegion(texture, 0, sourceRegion, desktopTexture, 0); - texture.Dispose(); - } - - - // render user view to seperate form - passThroughShader.viewport = new Viewport(0, 0, userViewForm.Width, userViewForm.Height); - // TODO: clean this up by simply using a pointer to the userViewSRV - if (threeDObjectEnabled) - { - passThroughShader.Render(deviceContext, userViewSRV, userViewForm.renderTargetView); - } - if (wobbleEffectEnabled) - { - passThroughShader.Render(deviceContext, filteredUserViewSRV, userViewForm.renderTargetView); - } - if (desktopDuplicationEnabled) - { - passThroughShader.Render(deviceContext, desktopTextureSRV, userViewForm.renderTargetView); - } - userViewForm.swapChain.Present(0, PresentFlags.None); - // projection puts x and y in [-1,1]; adjust to obtain texture coordinates [0,1] // TODO: put this in SetContants? userProjection[0, 0] /= 2; @@ -378,6 +157,7 @@ void RenderLoop() { var cameraDeviceResource = cameraDeviceResources[camera]; + // Copy camera pose var world = new SharpDX.Matrix(); for (int i = 0; i < 4; i++) for (int j = 0; j < 4; j++) @@ -387,421 +167,19 @@ void RenderLoop() var projectorWorldViewProjection = world * form.view * form.projection; var userWorldViewProjection = world * userView * userProjection; - projectiveTexturingShader.SetConstants(deviceContext, userWorldViewProjection, projectorWorldViewProjection); - - // TODO: clean this up by simply using a pointer to the userViewSRV - if (wobbleEffectEnabled) - projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, filteredUserViewSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport); - if (threeDObjectEnabled) - projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, userViewSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport); - if (desktopDuplicationEnabled) - projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, desktopTextureSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport); + projectiveTexturingShader.SetConstants(deviceContext, userWorldViewProjection, projectorWorldViewProjection, clock.Elapsed); + projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport); } form.swapChain.Present(1, PresentFlags.None); } - - if (desktopDuplicationEnabled) - outputDuplication.ReleaseFrame(); - - - Console.WriteLine(stopwatch.ElapsedMilliseconds); + //Console.WriteLine(stopwatch.ElapsedMilliseconds); stopwatch.Restart(); } } } - KinectSensor localKinectSensor; - BodyFrameReader bodyFrameReader; - Body[] bodies = null; - CameraSpacePoint headCameraSpacePoint, handLeftCameraSpacePoint, handRightCameraSpacePoint; - Object headCameraSpacePointLock = new Object(); - bool trackingValid = false; - - void LocalBodyLoop() - { - while (true) - { - // find closest tracked head - var bodyFrame = bodyFrameReader.AcquireLatestFrame(); - - if (bodyFrame != null) - { - if (bodies == null) - bodies = new Body[bodyFrame.BodyCount]; - bodyFrame.GetAndRefreshBodyData(bodies); - - bool foundTrackedBody = false; - float distanceToNearest = float.MaxValue; - var nearestHeadCameraSpacePoint = new CameraSpacePoint(); - var nearestHandRightCameraSpacePoint = new CameraSpacePoint(); - var nearestHandLeftCameraSpacePoint = new CameraSpacePoint(); - - - foreach (var body in bodies) - if (body.IsTracked) - { - var cameraSpacePoint = body.Joints[JointType.Head].Position; - if (cameraSpacePoint.Z < distanceToNearest) - { - distanceToNearest = cameraSpacePoint.Z; - nearestHeadCameraSpacePoint = cameraSpacePoint; - nearestHandLeftCameraSpacePoint = body.Joints[JointType.HandLeft].Position; - nearestHandRightCameraSpacePoint = body.Joints[JointType.HandRight].Position; - foundTrackedBody = true; - } - } - - lock (headCameraSpacePointLock) - { - if (foundTrackedBody) - { - headCameraSpacePoint = nearestHeadCameraSpacePoint; - handLeftCameraSpacePoint = nearestHandLeftCameraSpacePoint; - handRightCameraSpacePoint = nearestHandRightCameraSpacePoint; - - trackingValid = true; - //Console.WriteLine("{0} {1} {2}", headCameraSpacePoint.X, headCameraSpacePoint.Y, headCameraSpacePoint.Z); - } - else - { - headCameraSpacePoint.X = 0f; - headCameraSpacePoint.Y = 0.3f; - headCameraSpacePoint.Z = 1.5f; - - trackingValid = false; - } - } - - bodyFrame.Dispose(); - } - else - System.Threading.Thread.Sleep(5); - } - } - - - - - class CameraDeviceResource : IDisposable - { - // encapsulates d3d resources for a camera - public CameraDeviceResource(SharpDX.Direct3D11.Device device, ProjectorCameraEnsemble.Camera camera, Object renderLock, string directory) - { - this.device = device; - this.camera = camera; - this.renderLock = renderLock; - - // Kinect depth image - var depthImageTextureDesc = new Texture2DDescription() - { - Width = Kinect2Calibration.depthImageWidth, - Height = Kinect2Calibration.depthImageHeight, - MipLevels = 1, - ArraySize = 1, - Format = SharpDX.DXGI.Format.R16_UInt, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Dynamic, - BindFlags = BindFlags.ShaderResource, - CpuAccessFlags = CpuAccessFlags.Write, - }; - depthImageTexture = new Texture2D(device, depthImageTextureDesc); - depthImageTextureRV = new ShaderResourceView(device, depthImageTexture); - - var floatDepthImageTextureDesc = new Texture2DDescription() - { - Width = Kinect2Calibration.depthImageWidth, - Height = Kinect2Calibration.depthImageHeight, - MipLevels = 1, - ArraySize = 1, - Format = SharpDX.DXGI.Format.R32_Float, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Default, - BindFlags = BindFlags.RenderTarget | BindFlags.ShaderResource, - CpuAccessFlags = CpuAccessFlags.None, - }; - - floatDepthImageTexture = new Texture2D(device, floatDepthImageTextureDesc); - floatDepthImageRV = new ShaderResourceView(device, floatDepthImageTexture); - floatDepthImageRenderTargetView = new RenderTargetView(device, floatDepthImageTexture); - - floatDepthImageTexture2 = new Texture2D(device, floatDepthImageTextureDesc); - floatDepthImageRV2 = new ShaderResourceView(device, floatDepthImageTexture2); - floatDepthImageRenderTargetView2 = new RenderTargetView(device, floatDepthImageTexture2); - - // Kinect color image - var colorImageStagingTextureDesc = new Texture2DDescription() - { - Width = Kinect2Calibration.colorImageWidth, - Height = Kinect2Calibration.colorImageHeight, - MipLevels = 1, - ArraySize = 1, - Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Dynamic, - BindFlags = BindFlags.ShaderResource, - CpuAccessFlags = CpuAccessFlags.Write - }; - colorImageStagingTexture = new Texture2D(device, colorImageStagingTextureDesc); - - var colorImageTextureDesc = new Texture2DDescription() - { - Width = Kinect2Calibration.colorImageWidth, - Height = Kinect2Calibration.colorImageHeight, - MipLevels = 0, - ArraySize = 1, - Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Default, - BindFlags = BindFlags.ShaderResource | BindFlags.RenderTarget, - CpuAccessFlags = CpuAccessFlags.None, - OptionFlags = ResourceOptionFlags.GenerateMipMaps - }; - colorImageTexture = new Texture2D(device, colorImageTextureDesc); - colorImageTextureRV = new ShaderResourceView(device, colorImageTexture); - - // vertex buffer - var table = camera.calibration.ComputeDepthFrameToCameraSpaceTable(); - int numVertices = 6 * (Kinect2Calibration.depthImageWidth - 1) * (Kinect2Calibration.depthImageHeight - 1); - var vertices = new VertexPosition[numVertices]; - - Int3[] quadOffsets = new Int3[] - { - new Int3(0, 0, 0), - new Int3(1, 0, 0), - new Int3(0, 1, 0), - new Int3(1, 0, 0), - new Int3(1, 1, 0), - new Int3(0, 1, 0), - }; - - int vertexIndex = 0; - for (int y = 0; y < Kinect2Calibration.depthImageHeight - 1; y++) - for (int x = 0; x < Kinect2Calibration.depthImageWidth - 1; x++) - for (int i = 0; i < 6; i++) - { - int vertexX = x + quadOffsets[i].X; - int vertexY = y + quadOffsets[i].Y; - - var point = table[Kinect2Calibration.depthImageWidth * vertexY + vertexX]; - - var vertex = new VertexPosition(); - vertex.position = new SharpDX.Vector4(point.X, point.Y, vertexX, vertexY); - vertices[vertexIndex++] = vertex; - } - - var stream = new DataStream(numVertices * VertexPosition.SizeInBytes, true, true); - stream.WriteRange(vertices); - stream.Position = 0; - - var vertexBufferDesc = new BufferDescription() - { - BindFlags = BindFlags.VertexBuffer, - CpuAccessFlags = CpuAccessFlags.None, - Usage = ResourceUsage.Default, - SizeInBytes = numVertices * VertexPosition.SizeInBytes, - }; - vertexBuffer = new SharpDX.Direct3D11.Buffer(device, stream, vertexBufferDesc); - - vertexBufferBinding = new VertexBufferBinding(vertexBuffer, VertexPosition.SizeInBytes, 0); - - stream.Dispose(); - - var colorImage = new RoomAliveToolkit.ARGBImage(Kinect2Calibration.colorImageWidth, Kinect2Calibration.colorImageHeight); - ProjectorCameraEnsemble.LoadFromTiff(imagingFactory, colorImage, directory + "/camera" + camera.name + "/color.tiff"); - - var depthImage = new RoomAliveToolkit.ShortImage(Kinect2Calibration.depthImageWidth, Kinect2Calibration.depthImageHeight); - ProjectorCameraEnsemble.LoadFromTiff(imagingFactory, depthImage, directory + "/camera" + camera.name + "/mean.tiff"); - - lock (renderLock) // necessary? - { - UpdateColorImage(device.ImmediateContext, colorImage.DataIntPtr); - UpdateDepthImage(device.ImmediateContext, depthImage.DataIntPtr); - } - - colorImage.Dispose(); - depthImage.Dispose(); - - - - - } - - struct VertexPosition - { - public SharpDX.Vector4 position; - static public int SizeInBytes { get { return 4 * 4; } } - } - - public void Dispose() - { - depthImageTexture.Dispose(); - depthImageTextureRV.Dispose(); - colorImageTexture.Dispose(); - colorImageTextureRV.Dispose(); - colorImageStagingTexture.Dispose(); - vertexBuffer.Dispose(); - } - - SharpDX.Direct3D11.Device device; - public Texture2D depthImageTexture, floatDepthImageTexture, floatDepthImageTexture2; - public ShaderResourceView depthImageTextureRV, floatDepthImageRV, floatDepthImageRV2; - public RenderTargetView floatDepthImageRenderTargetView, floatDepthImageRenderTargetView2; - public Texture2D colorImageTexture; - public ShaderResourceView colorImageTextureRV; - public Texture2D colorImageStagingTexture; - public SharpDX.Direct3D11.Buffer vertexBuffer; - VertexBufferBinding vertexBufferBinding; - ProjectorCameraEnsemble.Camera camera; - public bool renderEnabled = true; - - public void UpdateDepthImage(DeviceContext deviceContext, IntPtr depthImage) - { - DataStream dataStream; - deviceContext.MapSubresource(depthImageTexture, 0, - MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); - dataStream.WriteRange(depthImage, Kinect2Calibration.depthImageWidth * Kinect2Calibration.depthImageHeight * 2); - deviceContext.UnmapSubresource(depthImageTexture, 0); - } - - public void UpdateDepthImage(DeviceContext deviceContext, byte[] depthImage) - { - DataStream dataStream; - deviceContext.MapSubresource(depthImageTexture, 0, - MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); - dataStream.WriteRange(depthImage, 0, Kinect2Calibration.depthImageWidth * Kinect2Calibration.depthImageHeight * 2); - deviceContext.UnmapSubresource(depthImageTexture, 0); - } - - public void UpdateColorImage(DeviceContext deviceContext, IntPtr colorImage) - { - DataStream dataStream; - deviceContext.MapSubresource(colorImageStagingTexture, 0, - MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); - dataStream.WriteRange(colorImage, Kinect2Calibration.colorImageWidth * Kinect2Calibration.colorImageHeight * 4); - deviceContext.UnmapSubresource(colorImageStagingTexture, 0); - - var resourceRegion = new ResourceRegion() - { - Left = 0, - Top = 0, - Right = Kinect2Calibration.colorImageWidth, - Bottom = Kinect2Calibration.colorImageHeight, - Front = 0, - Back = 1, - }; - deviceContext.CopySubresourceRegion(colorImageStagingTexture, 0, resourceRegion, colorImageTexture, 0); - deviceContext.GenerateMips(colorImageTextureRV); - } - - public void UpdateColorImage(DeviceContext deviceContext, byte[] colorImage) - { - DataStream dataStream; - deviceContext.MapSubresource(colorImageStagingTexture, 0, - MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); - dataStream.WriteRange(colorImage, 0, Kinect2Calibration.colorImageWidth * Kinect2Calibration.colorImageHeight * 4); - deviceContext.UnmapSubresource(colorImageStagingTexture, 0); - - var resourceRegion = new ResourceRegion() - { - Left = 0, - Top = 0, - Right = Kinect2Calibration.colorImageWidth, - Bottom = Kinect2Calibration.colorImageHeight, - Front = 0, - Back = 1, - }; - deviceContext.CopySubresourceRegion(colorImageStagingTexture, 0, resourceRegion, colorImageTexture, 0); - deviceContext.GenerateMips(colorImageTextureRV); - } - - public void Render(DeviceContext deviceContext) - { - deviceContext.InputAssembler.SetVertexBuffers(0, vertexBufferBinding); - deviceContext.VertexShader.SetShaderResource(0, depthImageTextureRV); - deviceContext.PixelShader.SetShaderResource(0, colorImageTextureRV); - deviceContext.Draw((Kinect2Calibration.depthImageWidth - 1) * (Kinect2Calibration.depthImageHeight - 1) * 6, 0); - } - - bool live = false; - - public void StartLive() - { - live = true; - //new System.Threading.Thread(ColorCameraLoop).Start(); - new System.Threading.Thread(DepthCameraLoop).Start(); - } - - public void StopLive() - { - live = false; - } - - - Object renderLock; - public bool depthImageChanged = true; - - //byte[] colorData = new byte[4 * Kinect2.Kinect2Calibration.colorImageWidth * Kinect2.Kinect2Calibration.colorImageHeight]; - byte[] nextColorData = new byte[4 * RoomAliveToolkit.Kinect2Calibration.colorImageWidth * RoomAliveToolkit.Kinect2Calibration.colorImageHeight]; - SharpDX.WIC.ImagingFactory2 imagingFactory = new SharpDX.WIC.ImagingFactory2(); - void ColorCameraLoop() - { - while (true) - { - var encodedColorData = camera.Client.LatestJPEGImage(); - - // decode JPEG - var memoryStream = new MemoryStream(encodedColorData); - var stream = new WICStream(imagingFactory, memoryStream); - // decodes to 24 bit BGR - var decoder = new SharpDX.WIC.BitmapDecoder(imagingFactory, stream, SharpDX.WIC.DecodeOptions.CacheOnLoad); - var bitmapFrameDecode = decoder.GetFrame(0); - - // convert to 32 bpp - var formatConverter = new FormatConverter(imagingFactory); - formatConverter.Initialize(bitmapFrameDecode, SharpDX.WIC.PixelFormat.Format32bppBGR); - formatConverter.CopyPixels(nextColorData, Kinect2Calibration.colorImageWidth * 4); // TODO: consider copying directly to texture native memory - //lock (colorData) - // Swap(ref colorData, ref nextColorData); - lock (renderLock) // necessary? - { - UpdateColorImage(device.ImmediateContext, nextColorData); - } - memoryStream.Close(); - memoryStream.Dispose(); - stream.Dispose(); - decoder.Dispose(); - formatConverter.Dispose(); - bitmapFrameDecode.Dispose(); - } - } - - //byte[] depthData = new byte[2 * Kinect2.Kinect2Calibration.depthImageWidth * Kinect2.Kinect2Calibration.depthImageHeight]; - byte[] nextDepthData; - void DepthCameraLoop() - { - while (true) - { - nextDepthData = camera.Client.LatestDepthImage(); - //lock (remoteDepthData) - // Swap(ref remoteDepthData, ref nextRemoteDepthData); - lock (renderLock) - { - depthImageChanged = true; - UpdateDepthImage(device.ImmediateContext, nextDepthData); - } - } - } - - static void Swap(ref T first, ref T second) - { - T temp = first; - first = second; - second = temp; - } - } - [DllImport("user32.dll", SetLastError = true)] static extern IntPtr FindWindow(string lpClassName, string lpWindowName); diff --git a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj index d10b8ab..8d99a73 100644 --- a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj +++ b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj @@ -77,6 +77,7 @@ + diff --git a/ProCamCalibration/ProjectionMappingSample/ProjectiveTexturingShader.cs b/ProCamCalibration/ProjectionMappingSample/ProjectiveTexturingShader.cs index 5452b52..3ea5493 100644 --- a/ProCamCalibration/ProjectionMappingSample/ProjectiveTexturingShader.cs +++ b/ProCamCalibration/ProjectionMappingSample/ProjectiveTexturingShader.cs @@ -2,6 +2,7 @@ using SharpDX.D3DCompiler; using SharpDX.Direct3D11; using SharpDX.DXGI; +using System; using System.IO; using System.Runtime.InteropServices; using Device = SharpDX.Direct3D11.Device; @@ -76,24 +77,33 @@ public ProjectiveTexturingShader(Device device) [StructLayout(LayoutKind.Explicit, Size = Constants.size)] public unsafe struct Constants { - public const int size = 128; + public const int size = 128 + 16; [FieldOffset(0)] public fixed float userWorldViewProjection[16]; [FieldOffset(64)] public fixed float projectorWorldViewProjection[16]; + [FieldOffset(128)] + public float globalTime; }; - public unsafe void SetConstants(DeviceContext deviceContext, SharpDX.Matrix userWorldViewProjection, SharpDX.Matrix projectorWorldViewProjection) + public unsafe void SetConstants(DeviceContext deviceContext, + SharpDX.Matrix userWorldViewProjection, + SharpDX.Matrix projectorWorldViewProjection, + TimeSpan elapsedTime) { Constants constants = new Constants(); for (int i = 0, col = 0; col < 4; col++) + { for (int row = 0; row < 4; row++) { constants.userWorldViewProjection[i] = userWorldViewProjection[row, col]; constants.projectorWorldViewProjection[i] = projectorWorldViewProjection[row, col]; i++; } + } + + constants.globalTime = (float)elapsedTime.TotalSeconds; DataStream dataStream; deviceContext.MapSubresource(constantBuffer, MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); @@ -101,7 +111,7 @@ public unsafe void SetConstants(DeviceContext deviceContext, SharpDX.Matrix user deviceContext.UnmapSubresource(constantBuffer, 0); } - public void Render(DeviceContext deviceContext, ShaderResourceView depthImageTextureRV, ShaderResourceView colorImageTextureRV, SharpDX.Direct3D11.Buffer vertexBuffer, RenderTargetView renderTargetView, DepthStencilView depthStencilView, Viewport viewport) + public void Render(DeviceContext deviceContext, ShaderResourceView depthImageTextureRV, SharpDX.Direct3D11.Buffer vertexBuffer, RenderTargetView renderTargetView, DepthStencilView depthStencilView, Viewport viewport) { deviceContext.InputAssembler.InputLayout = vertexInputLayout; deviceContext.InputAssembler.PrimitiveTopology = SharpDX.Direct3D.PrimitiveTopology.TriangleList; @@ -115,8 +125,7 @@ public void Render(DeviceContext deviceContext, ShaderResourceView depthImageTex deviceContext.VertexShader.SetConstantBuffer(0, constantBuffer); deviceContext.GeometryShader.Set(geometryShader); deviceContext.PixelShader.Set(pixelShader); - deviceContext.PixelShader.SetShaderResource(0, colorImageTextureRV); - deviceContext.PixelShader.SetSampler(0, colorSamplerState); + deviceContext.PixelShader.SetConstantBuffer(0, constantBuffer); deviceContext.Draw((Kinect2Calibration.depthImageWidth - 1) * (Kinect2Calibration.depthImageHeight - 1) * 6, 0); deviceContext.VertexShader.SetShaderResource(0, null); // to avoid warnings when these are later set as render targets diff --git a/ProCamCalibration/ProjectionMappingSample/Properties/Settings.Designer.cs b/ProCamCalibration/ProjectionMappingSample/Properties/Settings.Designer.cs index 0f4bcc0..c65978b 100644 --- a/ProCamCalibration/ProjectionMappingSample/Properties/Settings.Designer.cs +++ b/ProCamCalibration/ProjectionMappingSample/Properties/Settings.Designer.cs @@ -26,42 +26,6 @@ public static Settings Default { [global::System.Configuration.UserScopedSettingAttribute()] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Configuration.DefaultSettingValueAttribute("True")] - public bool ThreeDObjectEnabled { - get { - return ((bool)(this["ThreeDObjectEnabled"])); - } - set { - this["ThreeDObjectEnabled"] = value; - } - } - - [global::System.Configuration.UserScopedSettingAttribute()] - [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("False")] - public bool WobbleEffectEnabled { - get { - return ((bool)(this["WobbleEffectEnabled"])); - } - set { - this["WobbleEffectEnabled"] = value; - } - } - - [global::System.Configuration.UserScopedSettingAttribute()] - [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("False")] - public bool LocalHeadTrackingEnabled { - get { - return ((bool)(this["LocalHeadTrackingEnabled"])); - } - set { - this["LocalHeadTrackingEnabled"] = value; - } - } - - [global::System.Configuration.UserScopedSettingAttribute()] - [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("False")] public bool LiveDepthEnabled { get { return ((bool)(this["LiveDepthEnabled"])); @@ -73,7 +37,7 @@ public bool LiveDepthEnabled { [global::System.Configuration.UserScopedSettingAttribute()] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("False")] + [global::System.Configuration.DefaultSettingValueAttribute("True")] public bool FullScreenEnabled { get { return ((bool)(this["FullScreenEnabled"])); @@ -82,17 +46,5 @@ public bool FullScreenEnabled { this["FullScreenEnabled"] = value; } } - - [global::System.Configuration.UserScopedSettingAttribute()] - [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("False")] - public bool DesktopDuplicationEnabled { - get { - return ((bool)(this["DesktopDuplicationEnabled"])); - } - set { - this["DesktopDuplicationEnabled"] = value; - } - } } } diff --git a/ProCamCalibration/ProjectionMappingSample/Properties/Settings.settings b/ProCamCalibration/ProjectionMappingSample/Properties/Settings.settings index 9715df8..54a9f91 100644 --- a/ProCamCalibration/ProjectionMappingSample/Properties/Settings.settings +++ b/ProCamCalibration/ProjectionMappingSample/Properties/Settings.settings @@ -2,23 +2,11 @@ - - True - - - False - - - False - - False + True - False - - - False + True \ No newline at end of file diff --git a/ProCamCalibration/Shaders/DepthAndColorGS.hlsl b/ProCamCalibration/Shaders/DepthAndColorGS.hlsl index 9ab029f..437a2f7 100644 --- a/ProCamCalibration/Shaders/DepthAndColorGS.hlsl +++ b/ProCamCalibration/Shaders/DepthAndColorGS.hlsl @@ -3,6 +3,7 @@ struct GSInput float4 pos : SV_POSITION; float2 tex : TEXCOORD0; float depth : MYSEMANTIC; + float4 worldPos : TEXCOORD1; }; struct GSOutput @@ -10,6 +11,7 @@ struct GSOutput float4 pos : SV_POSITION; float2 tex : TEXCOORD0; float depth : MYSEMANTIC; + float4 worldPos : TEXCOORD1; }; [maxvertexcount(3)] diff --git a/ProCamCalibration/Shaders/DepthAndColorPS.hlsl b/ProCamCalibration/Shaders/DepthAndColorPS.hlsl index 733e86b..79c4517 100644 --- a/ProCamCalibration/Shaders/DepthAndColorPS.hlsl +++ b/ProCamCalibration/Shaders/DepthAndColorPS.hlsl @@ -1,14 +1,112 @@ -Texture2D colorTexture : register(t0); -SamplerState colorSampler : register(s0); struct PSInput { float4 pos : SV_POSITION; float2 tex : TEXCOORD0; float depth : MYSEMANTIC; + float4 worldPos : TEXCOORD1; }; -float4 main(PSInput input) : SV_TARGET0 +cbuffer constants : register(b0) +{ + // world is depth camera's pose; view and projection are that of the user + // also includes adjustment to go from view volume to texture coordinates + matrix userWorldViewProjection; + // world is depth camera's pose; view and projection are that of the projector + matrix projectorWorldViewProjection; + + float globalTime; +} + + +// Based on shaders: +// Template - 3D https://www.shadertoy.com/view/ldfSWs +// Xor - Triangle Grid, https://www.shadertoy.com/view/4tSGWz + +#define pi 3.14159265358979 +#define size 0.5 +#define reciproce_sqrt3 0.57735026918962576450914878050196 +#define lineThickness 0.01 + +float planeDistance = 0.2; +float offset; + +float r(float n) +{ + return frac(abs(sin(n*55.753)*367.34)); +} + +float r(float2 n) +{ + return r(dot(n, float2(2.46, -1.21))); +} + +float3 smallTrianglesColor(float3 pos) +{ + float a = (radians(60.0)); + float zoom = 0.125; + float2 c = (pos.xy + float2(0.0, pos.z)) * float2(sin(a), 1.0);//scaled coordinates + c = ((c + float2(c.y, 0.0)*cos(a)) / zoom) + float2(floor((c.x - c.y*cos(a)) / zoom*4.0) / 4.0, 0.0);//Add rotations + float type = (r(floor(c*4.0))*0.2 + r(floor(c*2.0))*0.3 + r(floor(c))*0.5);//Randomize type + type += 0.3 * sin(globalTime*5.0*type); + + float l = min(min((1.0 - (2.0 * abs(frac((c.x - c.y)*4.0) - 0.5))), + (1.0 - (2.0 * abs(frac(c.y * 4.0) - 0.5)))), + (1.0 - (2.0 * abs(frac(c.x * 4.0) - 0.5)))); + l = smoothstep(0.06, 0.04, l); + + return lerp(type, l, 0.3); +} + +float3 largeTrianglesColor(float3 pos) +{ + float a = (radians(60.0)); + float zoom = 0.5; + float2 c = (pos.xy + float2(0.0, pos.z)) * float2(sin(a), 1.0);//scaled coordinates + c = ((c + float2(c.y, 0.0)*cos(a)) / zoom) + float2(floor((c.x - c.y*cos(a)) / zoom*4.0) / 4.0, 0.0);//Add rotations + + float l = min(min((1.0 - (2.0 * abs(frac((c.x - c.y)*4.0) - 0.5))), + (1.0 - (2.0 * abs(frac(c.y * 4.0) - 0.5)))), + (1.0 - (2.0 * abs(frac(c.x * 4.0) - 0.5)))); + l = smoothstep(0.03, 0.02, l); + + return lerp(0.01, l, 1.0); +} + +//--------------------------------------------------------------- +// Material +// +// Defines the material (colors, shading, pattern, texturing) of the model +// at every point based on its position and normal. In this case, it simply +// returns a constant yellow color. +//------------------------------------------------------------------------ +float3 doMaterial(in float3 pos, in float3 midPos) { - return colorTexture.Sample(colorSampler, input.tex); + float d = length(pos.xz - midPos.xz) + pos.y - midPos.y; + + float border = fmod(globalTime, 5.0); + + float3 c1 = largeTrianglesColor(pos); + float3 c2 = smallTrianglesColor(pos); + + // Small rim + float3 c = float3(1.0, 1.0, 1.0) * smoothstep(border - 0.2, border, d); + // Large Triangles to all + c += c1; + // Small triangle slightly after front + c += c2 * smoothstep(border - 0.4, border - 0.6, d); + // Cutoff front + c *= smoothstep(border, border - 0.05, d); + // Cutoff back + c *= smoothstep(border - 3.0, border - 0.5, d); + // fade out + c *= smoothstep(5.0, 4.0, border); + + return c * float3(0.2, 0.5, 1.0); } + + +float4 main(PSInput input) : SV_TARGET0 +{ + return float4(doMaterial(input.worldPos.xyz, float3(0.0, -1.0, 3.0)), 1.0); +} \ No newline at end of file diff --git a/ProCamCalibration/Shaders/DepthAndProjectiveTextureVS.hlsl b/ProCamCalibration/Shaders/DepthAndProjectiveTextureVS.hlsl index d35b4cf..72fcaf9 100644 --- a/ProCamCalibration/Shaders/DepthAndProjectiveTextureVS.hlsl +++ b/ProCamCalibration/Shaders/DepthAndProjectiveTextureVS.hlsl @@ -10,6 +10,7 @@ struct VSOutput float4 pos : SV_POSITION; float2 tex : TEXCOORD0; float depth : MYSEMANTIC; + float4 worldPos : TEXCOORD1; }; cbuffer constants : register(b0) @@ -19,6 +20,8 @@ cbuffer constants : register(b0) matrix userWorldViewProjection; // world is depth camera's pose; view and projection are that of the projector matrix projectorWorldViewProjection; + + float globalTime; } // Input vertices are setup so that xy are the entries of the table @@ -52,6 +55,7 @@ VSOutput main(VSInput input) output.pos = pos; output.tex = userViewTex.xy; output.depth = depth; // passed to geometry shader to cull invalid triangles + output.worldPos = depthCamera; return output; } From 47583689bbd3886bfb945ffa789fdacd09c60ccf Mon Sep 17 00:00:00 2001 From: Roland Smeenk Date: Tue, 2 Feb 2016 14:05:41 +0100 Subject: [PATCH 3/9] Revert "Reverted to original after placing these changes on separate branch" This reverts commit 21544a2be853e54d77846c52f259546ca6394044. --- .../ProjectionMappingSample/App.config | 32 +- .../DepthAndColorShader.cs | 4 + .../ProjectionMappingSample/Form1.cs | 11 +- .../ProjectionMappingSample.cs | 648 +++++++++++++++++- .../ProjectionMappingSample.csproj | 1 - .../ProjectiveTexturingShader.cs | 19 +- .../Properties/Settings.Designer.cs | 50 +- .../Properties/Settings.settings | 16 +- .../Shaders/DepthAndColorGS.hlsl | 2 - .../Shaders/DepthAndColorPS.hlsl | 106 +-- .../Shaders/DepthAndProjectiveTextureVS.hlsl | 4 - 11 files changed, 734 insertions(+), 159 deletions(-) diff --git a/ProCamCalibration/ProjectionMappingSample/App.config b/ProCamCalibration/ProjectionMappingSample/App.config index 2223e7c..272bc19 100644 --- a/ProCamCalibration/ProjectionMappingSample/App.config +++ b/ProCamCalibration/ProjectionMappingSample/App.config @@ -8,14 +8,26 @@ - - - - True - - - True - - - + + + + True + + + False + + + False + + + False + + + False + + + False + + + \ No newline at end of file diff --git a/ProCamCalibration/ProjectionMappingSample/DepthAndColorShader.cs b/ProCamCalibration/ProjectionMappingSample/DepthAndColorShader.cs index f13eb8e..1a777aa 100644 --- a/ProCamCalibration/ProjectionMappingSample/DepthAndColorShader.cs +++ b/ProCamCalibration/ProjectionMappingSample/DepthAndColorShader.cs @@ -293,8 +293,12 @@ public void Render(DeviceContext deviceContext, ShaderResourceView depthImageTex SamplerState colorSamplerState; SharpDX.Direct3D11.Buffer constantBuffer; BilateralFilter bilateralFilter; + FromUIntPS fromUIntPS; Texture2D filteredDepthImageTexture, filteredDepthImageTexture2; RenderTargetView filteredRenderTargetView, filteredRenderTargetView2; ShaderResourceView filteredDepthImageSRV, filteredDepthImageSRV2; } + + + } diff --git a/ProCamCalibration/ProjectionMappingSample/Form1.cs b/ProCamCalibration/ProjectionMappingSample/Form1.cs index 1a40a92..165b6bb 100644 --- a/ProCamCalibration/ProjectionMappingSample/Form1.cs +++ b/ProCamCalibration/ProjectionMappingSample/Form1.cs @@ -133,16 +133,7 @@ public bool FullScreen // switch to fullscreen ShowInTaskbar = false; FormBorderStyle = FormBorderStyle.None; - - // Clamp to last screen if screen number is not available - if (projector.displayIndex >= Screen.AllScreens.Length) - { - Console.WriteLine("Projector '{0}' Clamped displayIndex '{1}' to '{2}'", projector.name, projector.displayIndex, Screen.AllScreens.Length - 1); - - projector.displayIndex = Screen.AllScreens.Length - 1; - } - - var bounds = Screen.AllScreens[projector.displayIndex].Bounds; + var bounds = Screen.AllScreens[projector.displayIndex].Bounds; // TODO: catch the case where the display is not available StartPosition = FormStartPosition.Manual; Location = new System.Drawing.Point(bounds.X, bounds.Y); Size = new Size(bounds.Width, bounds.Height); diff --git a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.cs b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.cs index cb5e62f..71354cd 100644 --- a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.cs +++ b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.cs @@ -8,8 +8,6 @@ using System.Windows.Forms; using Microsoft.Kinect; using System.Runtime.InteropServices; -using System.Diagnostics; -using System.Linq; namespace RoomAliveToolkit { @@ -39,6 +37,9 @@ public ProjectionMappingSample(string[] args) // shaders depthAndColorShader = new DepthAndColorShader(device); projectiveTexturingShader = new ProjectiveTexturingShader(device); + passThroughShader = new PassThrough(device, userViewTextureWidth, userViewTextureHeight); + radialWobbleShader = new RadialWobble(device, userViewTextureWidth, userViewTextureHeight); + meshShader = new MeshShader(device); fromUIntPS = new FromUIntPS(device, Kinect2Calibration.depthImageWidth, Kinect2Calibration.depthImageHeight); bilateralFilter = new BilateralFilter(device, Kinect2Calibration.depthImageWidth, Kinect2Calibration.depthImageHeight); @@ -46,6 +47,28 @@ public ProjectionMappingSample(string[] args) foreach (var camera in ensemble.cameras) cameraDeviceResources[camera] = new CameraDeviceResource(device, camera, renderLock, directory); + // one user view + // user view render target, depth buffer, viewport for user view + var userViewTextureDesc = new Texture2DDescription() + { + Width = userViewTextureWidth, + Height = userViewTextureHeight, + MipLevels = 1, // revisit this; we may benefit from mipmapping? + ArraySize = 1, + Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, + SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), + Usage = ResourceUsage.Default, + BindFlags = BindFlags.RenderTarget | BindFlags.ShaderResource, + CpuAccessFlags = CpuAccessFlags.None, + }; + var userViewRenderTarget = new Texture2D(device, userViewTextureDesc); + userViewRenderTargetView = new RenderTargetView(device, userViewRenderTarget); + userViewSRV = new ShaderResourceView(device, userViewRenderTarget); + + var filteredUserViewRenderTarget = new Texture2D(device, userViewTextureDesc); + filteredUserViewRenderTargetView = new RenderTargetView(device, filteredUserViewRenderTarget); + filteredUserViewSRV = new ShaderResourceView(device, filteredUserViewRenderTarget); + // user view depth buffer var userViewDpethBufferDesc = new Texture2DDescription() { @@ -62,6 +85,37 @@ public ProjectionMappingSample(string[] args) var userViewDepthStencil = new Texture2D(device, userViewDpethBufferDesc); userViewDepthStencilView = new DepthStencilView(device, userViewDepthStencil); + // user view viewport + userViewViewport = new Viewport(0, 0, userViewTextureWidth, userViewTextureHeight, 0f, 1f); + + // desktop region texture + + if (desktopDuplicationEnabled) + { + windowPtr = FindWindow(null, "Untitled - Notepad"); + + // maybe use FindWindowEx to find child window + + var rect = new RECT(); + GetWindowRect(windowPtr, out rect); + var desktopTextureDesc = new Texture2DDescription() + { + Width = (rect.Right + rightNudge) - (rect.Left + leftNudge), + Height = (rect.Bottom + bottomNudge) - (rect.Top + topNudge), + MipLevels = 1, // revisit this; we may benefit from mipmapping? + ArraySize = 1, + Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, + SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), + Usage = ResourceUsage.Default, + BindFlags = BindFlags.ShaderResource, + CpuAccessFlags = CpuAccessFlags.None, + }; + desktopTexture = new Texture2D(device, desktopTextureDesc); + desktopTextureSRV = new ShaderResourceView(device, desktopTexture); + } + + + // create a form for each projector foreach (var projector in ensemble.projectors) { @@ -72,7 +126,31 @@ public ProjectionMappingSample(string[] args) projectorForms.Add(form); } - clock.Start(); + // example 3d object + var mesh = Mesh.FromOBJFile("Content/FloorPlan.obj"); + meshDeviceResources = new MeshDeviceResources(device, imagingFactory, mesh); + + // desktop duplication + var output = new Output1(factory.Adapters[0].Outputs[0].NativePointer); // TODO: change adapter and output number + outputDuplication = output.DuplicateOutput(device); + + + userViewForm = new MainForm(factory, device, renderLock); + userViewForm.Show(); + + + userViewForm.videoPanel1.MouseClick += videoPanel1_MouseClick; + + // connect to local camera to acquire head position + if (localHeadTrackingEnabled) + { + localKinectSensor = KinectSensor.GetDefault(); + bodyFrameReader = localKinectSensor.BodyFrameSource.OpenReader(); + localKinectSensor.Open(); + Console.WriteLine("connected to local camera"); + new System.Threading.Thread(LocalBodyLoop).Start(); + } + if (liveDepthEnabled) { @@ -80,9 +158,19 @@ public ProjectionMappingSample(string[] args) cameraDeviceResource.StartLive(); } + new System.Threading.Thread(RenderLoop).Start(); } + void videoPanel1_MouseClick(object sender, MouseEventArgs e) + { + alpha = 0; + } + + + OutputDuplication outputDuplication; + + const int userViewTextureWidth = 2000; const int userViewTextureHeight = 1000; List projectorForms = new List(); @@ -90,39 +178,83 @@ public ProjectionMappingSample(string[] args) ProjectiveTexturingShader projectiveTexturingShader; Dictionary cameraDeviceResources = new Dictionary(); Object renderLock = new Object(); + RenderTargetView userViewRenderTargetView, filteredUserViewRenderTargetView; DepthStencilView userViewDepthStencilView; + ShaderResourceView userViewSRV, filteredUserViewSRV, desktopTextureSRV; + Viewport userViewViewport; SharpDX.Direct3D11.Device device; ProjectorCameraEnsemble ensemble; - + Form1 userViewForm; + MeshShader meshShader; + MeshDeviceResources meshDeviceResources; + PassThrough passThroughShader; + RadialWobble radialWobbleShader; FromUIntPS fromUIntPS; BilateralFilter bilateralFilter; System.Diagnostics.Stopwatch stopwatch = new System.Diagnostics.Stopwatch(); PointLight pointLight = new PointLight(); + Texture2D desktopTexture; + + IntPtr windowPtr; + const int leftNudge = 9; + const int topNudge = -10; + const int rightNudge = 0; + const int bottomNudge = 60; + SharpDX.WIC.ImagingFactory2 imagingFactory = new ImagingFactory2(); + float alpha = 1; // TODO: make so these can be changed live, put in menu + bool threeDObjectEnabled = Properties.Settings.Default.ThreeDObjectEnabled; + bool wobbleEffectEnabled = Properties.Settings.Default.WobbleEffectEnabled; + bool localHeadTrackingEnabled = Properties.Settings.Default.LocalHeadTrackingEnabled; bool liveDepthEnabled = Properties.Settings.Default.LiveDepthEnabled; bool fullScreenEnabled = Properties.Settings.Default.FullScreenEnabled; - - Stopwatch clock = new Stopwatch(); + bool desktopDuplicationEnabled = Properties.Settings.Default.DesktopDuplicationEnabled; void RenderLoop() { - while (true) + + while (true) { lock (renderLock) { var deviceContext = device.ImmediateContext; // render user view + deviceContext.ClearRenderTargetView(userViewRenderTargetView, Color4.Black); deviceContext.ClearDepthStencilView(userViewDepthStencilView, DepthStencilClearFlags.Depth, 1, 0); SharpDX.Vector3 headPosition = new SharpDX.Vector3(0f, 1.1f, -1.4f); // may need to change this default + if (localHeadTrackingEnabled) + { + float distanceSquared = 0; + lock (headCameraSpacePointLock) + { + headPosition = new SharpDX.Vector3(headCameraSpacePoint.X, headCameraSpacePoint.Y, headCameraSpacePoint.Z); + + float dx = handLeftCameraSpacePoint.X - handRightCameraSpacePoint.X; + float dy = handLeftCameraSpacePoint.Y - handRightCameraSpacePoint.Y; + float dz = handLeftCameraSpacePoint.Z - handRightCameraSpacePoint.Z; + distanceSquared = dx * dx + dy * dy + dz * dz; + } + var transform = SharpDX.Matrix.RotationY((float)Math.PI) * SharpDX.Matrix.Translation(-0.25f, 0.45f, 0); + headPosition = SharpDX.Vector3.TransformCoordinate(headPosition, transform); + + if (trackingValid && (distanceSquared < 0.02f) && (alpha > 1)) + alpha = 0; + //Console.WriteLine(distanceSquared); + } + var userView = GraphicsTransforms.LookAt(headPosition, headPosition + SharpDX.Vector3.UnitZ, SharpDX.Vector3.UnitY); userView.Transpose(); + + //Console.WriteLine("headPosition = " + headPosition); + + float aspect = (float)userViewTextureWidth / (float)userViewTextureHeight; var userProjection = GraphicsTransforms.PerspectiveFov(55.0f / 180.0f * (float)Math.PI, aspect, 0.001f, 1000.0f); userProjection.Transpose(); @@ -134,12 +266,101 @@ void RenderLoop() if (cameraDeviceResource.depthImageChanged) { fromUIntPS.Render(deviceContext, cameraDeviceResource.depthImageTextureRV, cameraDeviceResource.floatDepthImageRenderTargetView); - bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.floatDepthImageRenderTargetView2); - bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV2, cameraDeviceResource.floatDepthImageRenderTargetView); + for (int i = 0; i < 1; i++) + { + bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.floatDepthImageRenderTargetView2); + bilateralFilter.Render(deviceContext, cameraDeviceResource.floatDepthImageRV2, cameraDeviceResource.floatDepthImageRenderTargetView); + } cameraDeviceResource.depthImageChanged = false; } } + // wobble effect + if (wobbleEffectEnabled) + foreach (var camera in ensemble.cameras) + { + var cameraDeviceResource = cameraDeviceResources[camera]; + + var world = new SharpDX.Matrix(); + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + world[i, j] = (float)camera.pose[i, j]; + world.Transpose(); + + // view and projection matrix are post-multiply + var userWorldViewProjection = world * userView * userProjection; + + depthAndColorShader.SetConstants(deviceContext, camera.calibration, userWorldViewProjection); + depthAndColorShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.colorImageTextureRV, cameraDeviceResource.vertexBuffer, userViewRenderTargetView, userViewDepthStencilView, userViewViewport); + } + + // 3d object + if (threeDObjectEnabled) + { + var world = SharpDX.Matrix.Scaling(1.0f) * SharpDX.Matrix.RotationY(90.0f / 180.0f * (float)Math.PI) * + SharpDX.Matrix.RotationX(-40.0f / 180.0f * (float)Math.PI) * SharpDX.Matrix.Translation(0, 0.7f, 0.0f); + + var pointLight = new PointLight(); + pointLight.position = new Vector3(0, 2, 0); + pointLight.Ia = new Vector3(0.1f, 0.1f, 0.1f); + meshShader.SetVertexShaderConstants(deviceContext, world, userView * userProjection, pointLight.position); + meshShader.Render(deviceContext, meshDeviceResources, pointLight, userViewRenderTargetView, userViewDepthStencilView, userViewViewport); + } + + // wobble effect + if (wobbleEffectEnabled) + { + alpha += 0.05f; + if (alpha > 1) + radialWobbleShader.SetConstants(deviceContext, 0); + else + radialWobbleShader.SetConstants(deviceContext, alpha); + radialWobbleShader.Render(deviceContext, userViewSRV, filteredUserViewRenderTargetView); + } + + + // desktop duplication + if (desktopDuplicationEnabled) + { + var outputDuplicateFrameInformation = default(OutputDuplicateFrameInformation); + SharpDX.DXGI.Resource resource = null; + outputDuplication.AcquireNextFrame(1000, out outputDuplicateFrameInformation, out resource); + var texture = resource.QueryInterface(); + + var rect = new RECT(); + GetWindowRect(windowPtr, out rect); + + var sourceRegion = new ResourceRegion() + { + Left = rect.Left + leftNudge, + Right = rect.Right + rightNudge, + Top = rect.Top + topNudge, + Bottom = rect.Bottom + bottomNudge, + Front = 0, + Back = 1, + }; + deviceContext.CopySubresourceRegion(texture, 0, sourceRegion, desktopTexture, 0); + texture.Dispose(); + } + + + // render user view to seperate form + passThroughShader.viewport = new Viewport(0, 0, userViewForm.Width, userViewForm.Height); + // TODO: clean this up by simply using a pointer to the userViewSRV + if (threeDObjectEnabled) + { + passThroughShader.Render(deviceContext, userViewSRV, userViewForm.renderTargetView); + } + if (wobbleEffectEnabled) + { + passThroughShader.Render(deviceContext, filteredUserViewSRV, userViewForm.renderTargetView); + } + if (desktopDuplicationEnabled) + { + passThroughShader.Render(deviceContext, desktopTextureSRV, userViewForm.renderTargetView); + } + userViewForm.swapChain.Present(0, PresentFlags.None); + // projection puts x and y in [-1,1]; adjust to obtain texture coordinates [0,1] // TODO: put this in SetContants? userProjection[0, 0] /= 2; @@ -157,7 +378,6 @@ void RenderLoop() { var cameraDeviceResource = cameraDeviceResources[camera]; - // Copy camera pose var world = new SharpDX.Matrix(); for (int i = 0; i < 4; i++) for (int j = 0; j < 4; j++) @@ -167,19 +387,421 @@ void RenderLoop() var projectorWorldViewProjection = world * form.view * form.projection; var userWorldViewProjection = world * userView * userProjection; - projectiveTexturingShader.SetConstants(deviceContext, userWorldViewProjection, projectorWorldViewProjection, clock.Elapsed); - projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport); + projectiveTexturingShader.SetConstants(deviceContext, userWorldViewProjection, projectorWorldViewProjection); + + // TODO: clean this up by simply using a pointer to the userViewSRV + if (wobbleEffectEnabled) + projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, filteredUserViewSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport); + if (threeDObjectEnabled) + projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, userViewSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport); + if (desktopDuplicationEnabled) + projectiveTexturingShader.Render(deviceContext, cameraDeviceResource.floatDepthImageRV, desktopTextureSRV, cameraDeviceResource.vertexBuffer, form.renderTargetView, form.depthStencilView, form.viewport); } form.swapChain.Present(1, PresentFlags.None); } - //Console.WriteLine(stopwatch.ElapsedMilliseconds); + + if (desktopDuplicationEnabled) + outputDuplication.ReleaseFrame(); + + + Console.WriteLine(stopwatch.ElapsedMilliseconds); stopwatch.Restart(); } } } + KinectSensor localKinectSensor; + BodyFrameReader bodyFrameReader; + Body[] bodies = null; + CameraSpacePoint headCameraSpacePoint, handLeftCameraSpacePoint, handRightCameraSpacePoint; + Object headCameraSpacePointLock = new Object(); + bool trackingValid = false; + + void LocalBodyLoop() + { + while (true) + { + // find closest tracked head + var bodyFrame = bodyFrameReader.AcquireLatestFrame(); + + if (bodyFrame != null) + { + if (bodies == null) + bodies = new Body[bodyFrame.BodyCount]; + bodyFrame.GetAndRefreshBodyData(bodies); + + bool foundTrackedBody = false; + float distanceToNearest = float.MaxValue; + var nearestHeadCameraSpacePoint = new CameraSpacePoint(); + var nearestHandRightCameraSpacePoint = new CameraSpacePoint(); + var nearestHandLeftCameraSpacePoint = new CameraSpacePoint(); + + + foreach (var body in bodies) + if (body.IsTracked) + { + var cameraSpacePoint = body.Joints[JointType.Head].Position; + if (cameraSpacePoint.Z < distanceToNearest) + { + distanceToNearest = cameraSpacePoint.Z; + nearestHeadCameraSpacePoint = cameraSpacePoint; + nearestHandLeftCameraSpacePoint = body.Joints[JointType.HandLeft].Position; + nearestHandRightCameraSpacePoint = body.Joints[JointType.HandRight].Position; + foundTrackedBody = true; + } + } + + lock (headCameraSpacePointLock) + { + if (foundTrackedBody) + { + headCameraSpacePoint = nearestHeadCameraSpacePoint; + handLeftCameraSpacePoint = nearestHandLeftCameraSpacePoint; + handRightCameraSpacePoint = nearestHandRightCameraSpacePoint; + + trackingValid = true; + //Console.WriteLine("{0} {1} {2}", headCameraSpacePoint.X, headCameraSpacePoint.Y, headCameraSpacePoint.Z); + } + else + { + headCameraSpacePoint.X = 0f; + headCameraSpacePoint.Y = 0.3f; + headCameraSpacePoint.Z = 1.5f; + + trackingValid = false; + } + } + + bodyFrame.Dispose(); + } + else + System.Threading.Thread.Sleep(5); + } + } + + + + + class CameraDeviceResource : IDisposable + { + // encapsulates d3d resources for a camera + public CameraDeviceResource(SharpDX.Direct3D11.Device device, ProjectorCameraEnsemble.Camera camera, Object renderLock, string directory) + { + this.device = device; + this.camera = camera; + this.renderLock = renderLock; + + // Kinect depth image + var depthImageTextureDesc = new Texture2DDescription() + { + Width = Kinect2Calibration.depthImageWidth, + Height = Kinect2Calibration.depthImageHeight, + MipLevels = 1, + ArraySize = 1, + Format = SharpDX.DXGI.Format.R16_UInt, + SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), + Usage = ResourceUsage.Dynamic, + BindFlags = BindFlags.ShaderResource, + CpuAccessFlags = CpuAccessFlags.Write, + }; + depthImageTexture = new Texture2D(device, depthImageTextureDesc); + depthImageTextureRV = new ShaderResourceView(device, depthImageTexture); + + var floatDepthImageTextureDesc = new Texture2DDescription() + { + Width = Kinect2Calibration.depthImageWidth, + Height = Kinect2Calibration.depthImageHeight, + MipLevels = 1, + ArraySize = 1, + Format = SharpDX.DXGI.Format.R32_Float, + SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), + Usage = ResourceUsage.Default, + BindFlags = BindFlags.RenderTarget | BindFlags.ShaderResource, + CpuAccessFlags = CpuAccessFlags.None, + }; + + floatDepthImageTexture = new Texture2D(device, floatDepthImageTextureDesc); + floatDepthImageRV = new ShaderResourceView(device, floatDepthImageTexture); + floatDepthImageRenderTargetView = new RenderTargetView(device, floatDepthImageTexture); + + floatDepthImageTexture2 = new Texture2D(device, floatDepthImageTextureDesc); + floatDepthImageRV2 = new ShaderResourceView(device, floatDepthImageTexture2); + floatDepthImageRenderTargetView2 = new RenderTargetView(device, floatDepthImageTexture2); + + // Kinect color image + var colorImageStagingTextureDesc = new Texture2DDescription() + { + Width = Kinect2Calibration.colorImageWidth, + Height = Kinect2Calibration.colorImageHeight, + MipLevels = 1, + ArraySize = 1, + Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, + SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), + Usage = ResourceUsage.Dynamic, + BindFlags = BindFlags.ShaderResource, + CpuAccessFlags = CpuAccessFlags.Write + }; + colorImageStagingTexture = new Texture2D(device, colorImageStagingTextureDesc); + + var colorImageTextureDesc = new Texture2DDescription() + { + Width = Kinect2Calibration.colorImageWidth, + Height = Kinect2Calibration.colorImageHeight, + MipLevels = 0, + ArraySize = 1, + Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, + SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), + Usage = ResourceUsage.Default, + BindFlags = BindFlags.ShaderResource | BindFlags.RenderTarget, + CpuAccessFlags = CpuAccessFlags.None, + OptionFlags = ResourceOptionFlags.GenerateMipMaps + }; + colorImageTexture = new Texture2D(device, colorImageTextureDesc); + colorImageTextureRV = new ShaderResourceView(device, colorImageTexture); + + // vertex buffer + var table = camera.calibration.ComputeDepthFrameToCameraSpaceTable(); + int numVertices = 6 * (Kinect2Calibration.depthImageWidth - 1) * (Kinect2Calibration.depthImageHeight - 1); + var vertices = new VertexPosition[numVertices]; + + Int3[] quadOffsets = new Int3[] + { + new Int3(0, 0, 0), + new Int3(1, 0, 0), + new Int3(0, 1, 0), + new Int3(1, 0, 0), + new Int3(1, 1, 0), + new Int3(0, 1, 0), + }; + + int vertexIndex = 0; + for (int y = 0; y < Kinect2Calibration.depthImageHeight - 1; y++) + for (int x = 0; x < Kinect2Calibration.depthImageWidth - 1; x++) + for (int i = 0; i < 6; i++) + { + int vertexX = x + quadOffsets[i].X; + int vertexY = y + quadOffsets[i].Y; + + var point = table[Kinect2Calibration.depthImageWidth * vertexY + vertexX]; + + var vertex = new VertexPosition(); + vertex.position = new SharpDX.Vector4(point.X, point.Y, vertexX, vertexY); + vertices[vertexIndex++] = vertex; + } + + var stream = new DataStream(numVertices * VertexPosition.SizeInBytes, true, true); + stream.WriteRange(vertices); + stream.Position = 0; + + var vertexBufferDesc = new BufferDescription() + { + BindFlags = BindFlags.VertexBuffer, + CpuAccessFlags = CpuAccessFlags.None, + Usage = ResourceUsage.Default, + SizeInBytes = numVertices * VertexPosition.SizeInBytes, + }; + vertexBuffer = new SharpDX.Direct3D11.Buffer(device, stream, vertexBufferDesc); + + vertexBufferBinding = new VertexBufferBinding(vertexBuffer, VertexPosition.SizeInBytes, 0); + + stream.Dispose(); + + var colorImage = new RoomAliveToolkit.ARGBImage(Kinect2Calibration.colorImageWidth, Kinect2Calibration.colorImageHeight); + ProjectorCameraEnsemble.LoadFromTiff(imagingFactory, colorImage, directory + "/camera" + camera.name + "/color.tiff"); + + var depthImage = new RoomAliveToolkit.ShortImage(Kinect2Calibration.depthImageWidth, Kinect2Calibration.depthImageHeight); + ProjectorCameraEnsemble.LoadFromTiff(imagingFactory, depthImage, directory + "/camera" + camera.name + "/mean.tiff"); + + lock (renderLock) // necessary? + { + UpdateColorImage(device.ImmediateContext, colorImage.DataIntPtr); + UpdateDepthImage(device.ImmediateContext, depthImage.DataIntPtr); + } + + colorImage.Dispose(); + depthImage.Dispose(); + + + + + } + + struct VertexPosition + { + public SharpDX.Vector4 position; + static public int SizeInBytes { get { return 4 * 4; } } + } + + public void Dispose() + { + depthImageTexture.Dispose(); + depthImageTextureRV.Dispose(); + colorImageTexture.Dispose(); + colorImageTextureRV.Dispose(); + colorImageStagingTexture.Dispose(); + vertexBuffer.Dispose(); + } + + SharpDX.Direct3D11.Device device; + public Texture2D depthImageTexture, floatDepthImageTexture, floatDepthImageTexture2; + public ShaderResourceView depthImageTextureRV, floatDepthImageRV, floatDepthImageRV2; + public RenderTargetView floatDepthImageRenderTargetView, floatDepthImageRenderTargetView2; + public Texture2D colorImageTexture; + public ShaderResourceView colorImageTextureRV; + public Texture2D colorImageStagingTexture; + public SharpDX.Direct3D11.Buffer vertexBuffer; + VertexBufferBinding vertexBufferBinding; + ProjectorCameraEnsemble.Camera camera; + public bool renderEnabled = true; + + public void UpdateDepthImage(DeviceContext deviceContext, IntPtr depthImage) + { + DataStream dataStream; + deviceContext.MapSubresource(depthImageTexture, 0, + MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); + dataStream.WriteRange(depthImage, Kinect2Calibration.depthImageWidth * Kinect2Calibration.depthImageHeight * 2); + deviceContext.UnmapSubresource(depthImageTexture, 0); + } + + public void UpdateDepthImage(DeviceContext deviceContext, byte[] depthImage) + { + DataStream dataStream; + deviceContext.MapSubresource(depthImageTexture, 0, + MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); + dataStream.WriteRange(depthImage, 0, Kinect2Calibration.depthImageWidth * Kinect2Calibration.depthImageHeight * 2); + deviceContext.UnmapSubresource(depthImageTexture, 0); + } + + public void UpdateColorImage(DeviceContext deviceContext, IntPtr colorImage) + { + DataStream dataStream; + deviceContext.MapSubresource(colorImageStagingTexture, 0, + MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); + dataStream.WriteRange(colorImage, Kinect2Calibration.colorImageWidth * Kinect2Calibration.colorImageHeight * 4); + deviceContext.UnmapSubresource(colorImageStagingTexture, 0); + + var resourceRegion = new ResourceRegion() + { + Left = 0, + Top = 0, + Right = Kinect2Calibration.colorImageWidth, + Bottom = Kinect2Calibration.colorImageHeight, + Front = 0, + Back = 1, + }; + deviceContext.CopySubresourceRegion(colorImageStagingTexture, 0, resourceRegion, colorImageTexture, 0); + deviceContext.GenerateMips(colorImageTextureRV); + } + + public void UpdateColorImage(DeviceContext deviceContext, byte[] colorImage) + { + DataStream dataStream; + deviceContext.MapSubresource(colorImageStagingTexture, 0, + MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); + dataStream.WriteRange(colorImage, 0, Kinect2Calibration.colorImageWidth * Kinect2Calibration.colorImageHeight * 4); + deviceContext.UnmapSubresource(colorImageStagingTexture, 0); + + var resourceRegion = new ResourceRegion() + { + Left = 0, + Top = 0, + Right = Kinect2Calibration.colorImageWidth, + Bottom = Kinect2Calibration.colorImageHeight, + Front = 0, + Back = 1, + }; + deviceContext.CopySubresourceRegion(colorImageStagingTexture, 0, resourceRegion, colorImageTexture, 0); + deviceContext.GenerateMips(colorImageTextureRV); + } + + public void Render(DeviceContext deviceContext) + { + deviceContext.InputAssembler.SetVertexBuffers(0, vertexBufferBinding); + deviceContext.VertexShader.SetShaderResource(0, depthImageTextureRV); + deviceContext.PixelShader.SetShaderResource(0, colorImageTextureRV); + deviceContext.Draw((Kinect2Calibration.depthImageWidth - 1) * (Kinect2Calibration.depthImageHeight - 1) * 6, 0); + } + + bool live = false; + + public void StartLive() + { + live = true; + //new System.Threading.Thread(ColorCameraLoop).Start(); + new System.Threading.Thread(DepthCameraLoop).Start(); + } + + public void StopLive() + { + live = false; + } + + + Object renderLock; + public bool depthImageChanged = true; + + //byte[] colorData = new byte[4 * Kinect2.Kinect2Calibration.colorImageWidth * Kinect2.Kinect2Calibration.colorImageHeight]; + byte[] nextColorData = new byte[4 * RoomAliveToolkit.Kinect2Calibration.colorImageWidth * RoomAliveToolkit.Kinect2Calibration.colorImageHeight]; + SharpDX.WIC.ImagingFactory2 imagingFactory = new SharpDX.WIC.ImagingFactory2(); + void ColorCameraLoop() + { + while (true) + { + var encodedColorData = camera.Client.LatestJPEGImage(); + + // decode JPEG + var memoryStream = new MemoryStream(encodedColorData); + var stream = new WICStream(imagingFactory, memoryStream); + // decodes to 24 bit BGR + var decoder = new SharpDX.WIC.BitmapDecoder(imagingFactory, stream, SharpDX.WIC.DecodeOptions.CacheOnLoad); + var bitmapFrameDecode = decoder.GetFrame(0); + + // convert to 32 bpp + var formatConverter = new FormatConverter(imagingFactory); + formatConverter.Initialize(bitmapFrameDecode, SharpDX.WIC.PixelFormat.Format32bppBGR); + formatConverter.CopyPixels(nextColorData, Kinect2Calibration.colorImageWidth * 4); // TODO: consider copying directly to texture native memory + //lock (colorData) + // Swap(ref colorData, ref nextColorData); + lock (renderLock) // necessary? + { + UpdateColorImage(device.ImmediateContext, nextColorData); + } + memoryStream.Close(); + memoryStream.Dispose(); + stream.Dispose(); + decoder.Dispose(); + formatConverter.Dispose(); + bitmapFrameDecode.Dispose(); + } + } + + //byte[] depthData = new byte[2 * Kinect2.Kinect2Calibration.depthImageWidth * Kinect2.Kinect2Calibration.depthImageHeight]; + byte[] nextDepthData; + void DepthCameraLoop() + { + while (true) + { + nextDepthData = camera.Client.LatestDepthImage(); + //lock (remoteDepthData) + // Swap(ref remoteDepthData, ref nextRemoteDepthData); + lock (renderLock) + { + depthImageChanged = true; + UpdateDepthImage(device.ImmediateContext, nextDepthData); + } + } + } + + static void Swap(ref T first, ref T second) + { + T temp = first; + first = second; + second = temp; + } + } + [DllImport("user32.dll", SetLastError = true)] static extern IntPtr FindWindow(string lpClassName, string lpWindowName); diff --git a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj index 8d99a73..d10b8ab 100644 --- a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj +++ b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj @@ -77,7 +77,6 @@ - diff --git a/ProCamCalibration/ProjectionMappingSample/ProjectiveTexturingShader.cs b/ProCamCalibration/ProjectionMappingSample/ProjectiveTexturingShader.cs index 3ea5493..5452b52 100644 --- a/ProCamCalibration/ProjectionMappingSample/ProjectiveTexturingShader.cs +++ b/ProCamCalibration/ProjectionMappingSample/ProjectiveTexturingShader.cs @@ -2,7 +2,6 @@ using SharpDX.D3DCompiler; using SharpDX.Direct3D11; using SharpDX.DXGI; -using System; using System.IO; using System.Runtime.InteropServices; using Device = SharpDX.Direct3D11.Device; @@ -77,33 +76,24 @@ public ProjectiveTexturingShader(Device device) [StructLayout(LayoutKind.Explicit, Size = Constants.size)] public unsafe struct Constants { - public const int size = 128 + 16; + public const int size = 128; [FieldOffset(0)] public fixed float userWorldViewProjection[16]; [FieldOffset(64)] public fixed float projectorWorldViewProjection[16]; - [FieldOffset(128)] - public float globalTime; }; - public unsafe void SetConstants(DeviceContext deviceContext, - SharpDX.Matrix userWorldViewProjection, - SharpDX.Matrix projectorWorldViewProjection, - TimeSpan elapsedTime) + public unsafe void SetConstants(DeviceContext deviceContext, SharpDX.Matrix userWorldViewProjection, SharpDX.Matrix projectorWorldViewProjection) { Constants constants = new Constants(); for (int i = 0, col = 0; col < 4; col++) - { for (int row = 0; row < 4; row++) { constants.userWorldViewProjection[i] = userWorldViewProjection[row, col]; constants.projectorWorldViewProjection[i] = projectorWorldViewProjection[row, col]; i++; } - } - - constants.globalTime = (float)elapsedTime.TotalSeconds; DataStream dataStream; deviceContext.MapSubresource(constantBuffer, MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); @@ -111,7 +101,7 @@ public unsafe void SetConstants(DeviceContext deviceContext, deviceContext.UnmapSubresource(constantBuffer, 0); } - public void Render(DeviceContext deviceContext, ShaderResourceView depthImageTextureRV, SharpDX.Direct3D11.Buffer vertexBuffer, RenderTargetView renderTargetView, DepthStencilView depthStencilView, Viewport viewport) + public void Render(DeviceContext deviceContext, ShaderResourceView depthImageTextureRV, ShaderResourceView colorImageTextureRV, SharpDX.Direct3D11.Buffer vertexBuffer, RenderTargetView renderTargetView, DepthStencilView depthStencilView, Viewport viewport) { deviceContext.InputAssembler.InputLayout = vertexInputLayout; deviceContext.InputAssembler.PrimitiveTopology = SharpDX.Direct3D.PrimitiveTopology.TriangleList; @@ -125,7 +115,8 @@ public void Render(DeviceContext deviceContext, ShaderResourceView depthImageTex deviceContext.VertexShader.SetConstantBuffer(0, constantBuffer); deviceContext.GeometryShader.Set(geometryShader); deviceContext.PixelShader.Set(pixelShader); - deviceContext.PixelShader.SetConstantBuffer(0, constantBuffer); + deviceContext.PixelShader.SetShaderResource(0, colorImageTextureRV); + deviceContext.PixelShader.SetSampler(0, colorSamplerState); deviceContext.Draw((Kinect2Calibration.depthImageWidth - 1) * (Kinect2Calibration.depthImageHeight - 1) * 6, 0); deviceContext.VertexShader.SetShaderResource(0, null); // to avoid warnings when these are later set as render targets diff --git a/ProCamCalibration/ProjectionMappingSample/Properties/Settings.Designer.cs b/ProCamCalibration/ProjectionMappingSample/Properties/Settings.Designer.cs index c65978b..0f4bcc0 100644 --- a/ProCamCalibration/ProjectionMappingSample/Properties/Settings.Designer.cs +++ b/ProCamCalibration/ProjectionMappingSample/Properties/Settings.Designer.cs @@ -26,6 +26,42 @@ public static Settings Default { [global::System.Configuration.UserScopedSettingAttribute()] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Configuration.DefaultSettingValueAttribute("True")] + public bool ThreeDObjectEnabled { + get { + return ((bool)(this["ThreeDObjectEnabled"])); + } + set { + this["ThreeDObjectEnabled"] = value; + } + } + + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("False")] + public bool WobbleEffectEnabled { + get { + return ((bool)(this["WobbleEffectEnabled"])); + } + set { + this["WobbleEffectEnabled"] = value; + } + } + + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("False")] + public bool LocalHeadTrackingEnabled { + get { + return ((bool)(this["LocalHeadTrackingEnabled"])); + } + set { + this["LocalHeadTrackingEnabled"] = value; + } + } + + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("False")] public bool LiveDepthEnabled { get { return ((bool)(this["LiveDepthEnabled"])); @@ -37,7 +73,7 @@ public bool LiveDepthEnabled { [global::System.Configuration.UserScopedSettingAttribute()] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] - [global::System.Configuration.DefaultSettingValueAttribute("True")] + [global::System.Configuration.DefaultSettingValueAttribute("False")] public bool FullScreenEnabled { get { return ((bool)(this["FullScreenEnabled"])); @@ -46,5 +82,17 @@ public bool FullScreenEnabled { this["FullScreenEnabled"] = value; } } + + [global::System.Configuration.UserScopedSettingAttribute()] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Configuration.DefaultSettingValueAttribute("False")] + public bool DesktopDuplicationEnabled { + get { + return ((bool)(this["DesktopDuplicationEnabled"])); + } + set { + this["DesktopDuplicationEnabled"] = value; + } + } } } diff --git a/ProCamCalibration/ProjectionMappingSample/Properties/Settings.settings b/ProCamCalibration/ProjectionMappingSample/Properties/Settings.settings index 54a9f91..9715df8 100644 --- a/ProCamCalibration/ProjectionMappingSample/Properties/Settings.settings +++ b/ProCamCalibration/ProjectionMappingSample/Properties/Settings.settings @@ -2,11 +2,23 @@ - + True + + False + + + False + + + False + - True + False + + + False \ No newline at end of file diff --git a/ProCamCalibration/Shaders/DepthAndColorGS.hlsl b/ProCamCalibration/Shaders/DepthAndColorGS.hlsl index 437a2f7..9ab029f 100644 --- a/ProCamCalibration/Shaders/DepthAndColorGS.hlsl +++ b/ProCamCalibration/Shaders/DepthAndColorGS.hlsl @@ -3,7 +3,6 @@ struct GSInput float4 pos : SV_POSITION; float2 tex : TEXCOORD0; float depth : MYSEMANTIC; - float4 worldPos : TEXCOORD1; }; struct GSOutput @@ -11,7 +10,6 @@ struct GSOutput float4 pos : SV_POSITION; float2 tex : TEXCOORD0; float depth : MYSEMANTIC; - float4 worldPos : TEXCOORD1; }; [maxvertexcount(3)] diff --git a/ProCamCalibration/Shaders/DepthAndColorPS.hlsl b/ProCamCalibration/Shaders/DepthAndColorPS.hlsl index 79c4517..733e86b 100644 --- a/ProCamCalibration/Shaders/DepthAndColorPS.hlsl +++ b/ProCamCalibration/Shaders/DepthAndColorPS.hlsl @@ -1,112 +1,14 @@ +Texture2D colorTexture : register(t0); +SamplerState colorSampler : register(s0); struct PSInput { float4 pos : SV_POSITION; float2 tex : TEXCOORD0; float depth : MYSEMANTIC; - float4 worldPos : TEXCOORD1; }; -cbuffer constants : register(b0) -{ - // world is depth camera's pose; view and projection are that of the user - // also includes adjustment to go from view volume to texture coordinates - matrix userWorldViewProjection; - // world is depth camera's pose; view and projection are that of the projector - matrix projectorWorldViewProjection; - - float globalTime; -} - - -// Based on shaders: -// Template - 3D https://www.shadertoy.com/view/ldfSWs -// Xor - Triangle Grid, https://www.shadertoy.com/view/4tSGWz - -#define pi 3.14159265358979 -#define size 0.5 -#define reciproce_sqrt3 0.57735026918962576450914878050196 -#define lineThickness 0.01 - -float planeDistance = 0.2; -float offset; - -float r(float n) -{ - return frac(abs(sin(n*55.753)*367.34)); -} - -float r(float2 n) -{ - return r(dot(n, float2(2.46, -1.21))); -} - -float3 smallTrianglesColor(float3 pos) -{ - float a = (radians(60.0)); - float zoom = 0.125; - float2 c = (pos.xy + float2(0.0, pos.z)) * float2(sin(a), 1.0);//scaled coordinates - c = ((c + float2(c.y, 0.0)*cos(a)) / zoom) + float2(floor((c.x - c.y*cos(a)) / zoom*4.0) / 4.0, 0.0);//Add rotations - float type = (r(floor(c*4.0))*0.2 + r(floor(c*2.0))*0.3 + r(floor(c))*0.5);//Randomize type - type += 0.3 * sin(globalTime*5.0*type); - - float l = min(min((1.0 - (2.0 * abs(frac((c.x - c.y)*4.0) - 0.5))), - (1.0 - (2.0 * abs(frac(c.y * 4.0) - 0.5)))), - (1.0 - (2.0 * abs(frac(c.x * 4.0) - 0.5)))); - l = smoothstep(0.06, 0.04, l); - - return lerp(type, l, 0.3); -} - -float3 largeTrianglesColor(float3 pos) -{ - float a = (radians(60.0)); - float zoom = 0.5; - float2 c = (pos.xy + float2(0.0, pos.z)) * float2(sin(a), 1.0);//scaled coordinates - c = ((c + float2(c.y, 0.0)*cos(a)) / zoom) + float2(floor((c.x - c.y*cos(a)) / zoom*4.0) / 4.0, 0.0);//Add rotations - - float l = min(min((1.0 - (2.0 * abs(frac((c.x - c.y)*4.0) - 0.5))), - (1.0 - (2.0 * abs(frac(c.y * 4.0) - 0.5)))), - (1.0 - (2.0 * abs(frac(c.x * 4.0) - 0.5)))); - l = smoothstep(0.03, 0.02, l); - - return lerp(0.01, l, 1.0); -} - -//--------------------------------------------------------------- -// Material -// -// Defines the material (colors, shading, pattern, texturing) of the model -// at every point based on its position and normal. In this case, it simply -// returns a constant yellow color. -//------------------------------------------------------------------------ -float3 doMaterial(in float3 pos, in float3 midPos) -{ - float d = length(pos.xz - midPos.xz) + pos.y - midPos.y; - - float border = fmod(globalTime, 5.0); - - float3 c1 = largeTrianglesColor(pos); - float3 c2 = smallTrianglesColor(pos); - - // Small rim - float3 c = float3(1.0, 1.0, 1.0) * smoothstep(border - 0.2, border, d); - // Large Triangles to all - c += c1; - // Small triangle slightly after front - c += c2 * smoothstep(border - 0.4, border - 0.6, d); - // Cutoff front - c *= smoothstep(border, border - 0.05, d); - // Cutoff back - c *= smoothstep(border - 3.0, border - 0.5, d); - // fade out - c *= smoothstep(5.0, 4.0, border); - - return c * float3(0.2, 0.5, 1.0); -} - - float4 main(PSInput input) : SV_TARGET0 { - return float4(doMaterial(input.worldPos.xyz, float3(0.0, -1.0, 3.0)), 1.0); -} \ No newline at end of file + return colorTexture.Sample(colorSampler, input.tex); +} diff --git a/ProCamCalibration/Shaders/DepthAndProjectiveTextureVS.hlsl b/ProCamCalibration/Shaders/DepthAndProjectiveTextureVS.hlsl index 72fcaf9..d35b4cf 100644 --- a/ProCamCalibration/Shaders/DepthAndProjectiveTextureVS.hlsl +++ b/ProCamCalibration/Shaders/DepthAndProjectiveTextureVS.hlsl @@ -10,7 +10,6 @@ struct VSOutput float4 pos : SV_POSITION; float2 tex : TEXCOORD0; float depth : MYSEMANTIC; - float4 worldPos : TEXCOORD1; }; cbuffer constants : register(b0) @@ -20,8 +19,6 @@ cbuffer constants : register(b0) matrix userWorldViewProjection; // world is depth camera's pose; view and projection are that of the projector matrix projectorWorldViewProjection; - - float globalTime; } // Input vertices are setup so that xy are the entries of the table @@ -55,7 +52,6 @@ VSOutput main(VSInput input) output.pos = pos; output.tex = userViewTex.xy; output.depth = depth; // passed to geometry shader to cull invalid triangles - output.worldPos = depthCamera; return output; } From cea59d3ffe98fa5ba58ef9320c7841d1cee30f77 Mon Sep 17 00:00:00 2001 From: Roland Smeenk Date: Tue, 2 Feb 2016 15:11:04 +0100 Subject: [PATCH 4/9] Projects now use SharpDXDirectXVersion to automatically resolve the correct SharpDX assemblies. Added BringToFront to projector server. Implemented wireframe toggle. Added camera and projector frustum menu toggles. --- .../CalibrateEnsemble.csproj | 13 +++-- .../CalibrateEnsemble/MainForm.Designer.cs | 34 ++++++++++- .../CalibrateEnsemble/MainForm.cs | 56 +++++++++++++++---- .../KinectServer/KinectServer.csproj | 7 ++- .../ProCamEnsembleCalibrationLib.csproj | 26 +++++---- .../ProjectionMappingSample.csproj | 18 +++--- .../ProjectorServer/ProjectorServer.cs | 2 + .../ProjectorServer/ProjectorServer.csproj | 9 +-- ProCamCalibration/Shaders/Shaders.vcxproj | 2 +- 9 files changed, 122 insertions(+), 45 deletions(-) diff --git a/ProCamCalibration/CalibrateEnsemble/CalibrateEnsemble.csproj b/ProCamCalibration/CalibrateEnsemble/CalibrateEnsemble.csproj index 80a470d..17d25da 100644 --- a/ProCamCalibration/CalibrateEnsemble/CalibrateEnsemble.csproj +++ b/ProCamCalibration/CalibrateEnsemble/CalibrateEnsemble.csproj @@ -15,6 +15,7 @@ SAK SAK SAK + DirectX11_2 1eec9909 @@ -28,7 +29,7 @@ 4 true false - true + false AnyCPU @@ -73,23 +74,23 @@ False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.dll + $(SharpDXPackageBinDir)\SharpDX.dll False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.D3DCompiler.dll + $(SharpDXPackageBinDir)\SharpDX.D3DCompiler.dll False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct2D1.dll + $(SharpDXPackageBinDir)\SharpDX.Direct2D1.dll False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct3D11.dll + $(SharpDXPackageBinDir)\SharpDX.Direct3D11.dll False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.DXGI.dll + $(SharpDXPackageBinDir)\SharpDX.DXGI.dll diff --git a/ProCamCalibration/CalibrateEnsemble/MainForm.Designer.cs b/ProCamCalibration/CalibrateEnsemble/MainForm.Designer.cs index 156c1a7..b23e94a 100644 --- a/ProCamCalibration/CalibrateEnsemble/MainForm.Designer.cs +++ b/ProCamCalibration/CalibrateEnsemble/MainForm.Designer.cs @@ -52,6 +52,9 @@ private void InitializeComponent() this.renderToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.liveViewToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.toolStripSeparator2 = new System.Windows.Forms.ToolStripSeparator(); + this.wireframeToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + this.cameraFrustumsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + this.projectorFrustumsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.viewToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.perspectiveAtOriginToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.toolStripSeparator3 = new System.Windows.Forms.ToolStripSeparator(); @@ -239,6 +242,9 @@ private void InitializeComponent() // this.renderToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { this.liveViewToolStripMenuItem, + this.wireframeToolStripMenuItem, + this.cameraFrustumsToolStripMenuItem, + this.projectorFrustumsToolStripMenuItem, this.toolStripSeparator2}); this.renderToolStripMenuItem.Name = "renderToolStripMenuItem"; this.renderToolStripMenuItem.Size = new System.Drawing.Size(56, 20); @@ -247,14 +253,35 @@ private void InitializeComponent() // liveViewToolStripMenuItem // this.liveViewToolStripMenuItem.Name = "liveViewToolStripMenuItem"; - this.liveViewToolStripMenuItem.Size = new System.Drawing.Size(123, 22); + this.liveViewToolStripMenuItem.Size = new System.Drawing.Size(180, 22); this.liveViewToolStripMenuItem.Text = "Live View"; this.liveViewToolStripMenuItem.Click += new System.EventHandler(this.liveViewToolStripMenuItem_Click); // // toolStripSeparator2 // this.toolStripSeparator2.Name = "toolStripSeparator2"; - this.toolStripSeparator2.Size = new System.Drawing.Size(120, 6); + this.toolStripSeparator2.Size = new System.Drawing.Size(177, 6); + // + // wireframeToolStripMenuItem + // + this.wireframeToolStripMenuItem.Name = "wireframeToolStripMenuItem"; + this.wireframeToolStripMenuItem.Size = new System.Drawing.Size(74, 20); + this.wireframeToolStripMenuItem.Text = "Wireframe"; + this.wireframeToolStripMenuItem.Click += new System.EventHandler(this.wireframeViewToolStripMenuItem_Click); + // + // cameraFrustumsToolStripMenuItem + // + this.cameraFrustumsToolStripMenuItem.Name = "cameraFrustumsToolStripMenuItem"; + this.cameraFrustumsToolStripMenuItem.Size = new System.Drawing.Size(112, 20); + this.cameraFrustumsToolStripMenuItem.Text = "Camera Frustums"; + this.cameraFrustumsToolStripMenuItem.Click += new System.EventHandler(this.cameraFrustumToolStripMenuItem_Click); + // + // projectorFrustumsToolStripMenuItem + // + this.projectorFrustumsToolStripMenuItem.Name = "projectorFrustumsToolStripMenuItem"; + this.projectorFrustumsToolStripMenuItem.Size = new System.Drawing.Size(119, 20); + this.projectorFrustumsToolStripMenuItem.Text = "Projector Frustums"; + this.projectorFrustumsToolStripMenuItem.Click += new System.EventHandler(this.projectorFrustumToolStripMenuItem_Click); // // viewToolStripMenuItem // @@ -402,5 +429,8 @@ private void InitializeComponent() private System.Windows.Forms.ToolStripMenuItem decodeGrayCodesToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem calibrateProjectorGroupsToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem optimizePoseToolStripMenuItem; + private System.Windows.Forms.ToolStripMenuItem wireframeToolStripMenuItem; + private System.Windows.Forms.ToolStripMenuItem cameraFrustumsToolStripMenuItem; + private System.Windows.Forms.ToolStripMenuItem projectorFrustumsToolStripMenuItem; } } \ No newline at end of file diff --git a/ProCamCalibration/CalibrateEnsemble/MainForm.cs b/ProCamCalibration/CalibrateEnsemble/MainForm.cs index 9feddfe..6c48f32 100644 --- a/ProCamCalibration/CalibrateEnsemble/MainForm.cs +++ b/ProCamCalibration/CalibrateEnsemble/MainForm.cs @@ -93,16 +93,7 @@ protected override void OnLoad(EventArgs e) }; depthStencilState = new DepthStencilState(device, depthStencilStateDesc); - // rasterizer states - var rasterizerStateDesc = new RasterizerStateDescription() - { - CullMode = CullMode.None, // beware what this does to both shaders - FillMode = FillMode.Solid, - IsDepthClipEnabled = true, - IsFrontCounterClockwise = true, - IsMultisampleEnabled = true, - }; - rasterizerState = new RasterizerState(device, rasterizerStateDesc); + UpdateRasterizerState(); // color sampler state var colorSamplerStateDesc = new SamplerStateDescription() @@ -160,6 +151,23 @@ protected override void OnLoad(EventArgs e) new System.Threading.Thread(RenderLoop).Start(); } + private void UpdateRasterizerState() + { + lock (renderLock) + { + // rasterizer states + var rasterizerStateDesc = new RasterizerStateDescription() + { + CullMode = CullMode.None, // beware what this does to both shaders + FillMode = wireframe ? FillMode.Wireframe : FillMode.Solid, + IsDepthClipEnabled = true, + IsFrontCounterClockwise = true, + IsMultisampleEnabled = true, + }; + rasterizerState = new RasterizerState(device, rasterizerStateDesc); + } + } + SharpDX.Direct3D11.Device device; SwapChain swapChain; Texture2D renderTarget, depthStencil; @@ -502,12 +510,16 @@ public unsafe void SetConstants(DeviceContext deviceContext, RoomAliveToolkit.Ki // hlsl matrices are default column order var constants = new ConstantBuffer(); for (int i = 0, col = 0; col < 4; col++) + { for (int row = 0; row < 4; row++) { constants.projection[i] = projection[row, col]; constants.depthToColorTransform[i] = (float)kinect2Calibration.depthToColorTransform[row, col]; i++; } + } + + constants.f[0] = (float)kinect2Calibration.colorCameraMatrix[0, 0]; constants.f[1] = (float)kinect2Calibration.colorCameraMatrix[1, 1]; constants.c[0] = (float)kinect2Calibration.colorCameraMatrix[0, 2]; @@ -533,7 +545,6 @@ void RenderLoop() view = manipulator.Update(); var deviceContext = device.ImmediateContext; - deviceContext.InputAssembler.InputLayout = vertexInputLayout; deviceContext.InputAssembler.PrimitiveTopology = SharpDX.Direct3D.PrimitiveTopology.TriangleList; deviceContext.OutputMerger.SetTargets(depthStencilView, renderTargetView); @@ -909,6 +920,28 @@ private void liveViewToolStripMenuItem_Click(object sender, EventArgs e) liveViewToolStripMenuItem.Checked = live; } + bool wireframe = false; + private void wireframeViewToolStripMenuItem_Click(object sender, EventArgs e) + { + wireframe = !wireframe; + wireframeToolStripMenuItem.Checked = wireframe; + UpdateRasterizerState(); + } + + bool renderCameraFrustums = false; + private void cameraFrustumToolStripMenuItem_Click(object sender, EventArgs e) + { + renderCameraFrustums = !renderCameraFrustums; + cameraFrustumsToolStripMenuItem.Checked = renderCameraFrustums; + } + + bool renderProjectorFrustums = false; + private void projectorFrustumToolStripMenuItem_Click(object sender, EventArgs e) + { + renderProjectorFrustums = !renderProjectorFrustums; + projectorFrustumsToolStripMenuItem.Checked = renderCameraFrustums; + } + private void videoPanel1_SizeChanged(object sender, EventArgs e) { // TODO: look into using this as initial creation @@ -1094,6 +1127,7 @@ void LoadEnsemble() { ensemble = ProjectorCameraEnsemble.FromFile(path); Console.WriteLine("Loaded " + path); + Console.WriteLine("Containing: {0} camera(s), {1} projector(s)", ensemble.cameras.Count, ensemble.projectors.Count); } catch (Exception ex) { diff --git a/ProCamCalibration/KinectServer/KinectServer.csproj b/ProCamCalibration/KinectServer/KinectServer.csproj index 44d02d5..17b2968 100644 --- a/ProCamCalibration/KinectServer/KinectServer.csproj +++ b/ProCamCalibration/KinectServer/KinectServer.csproj @@ -16,6 +16,7 @@ SAK SAK + DirectX11_2 0eef0576 @@ -27,7 +28,7 @@ DEBUG;TRACE prompt 4 - true + false false @@ -69,11 +70,11 @@ False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.dll + $(SharpDXPackageBinDir)\SharpDX.dll False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct2D1.dll + $(SharpDXPackageBinDir)\SharpDX.Direct2D1.dll diff --git a/ProCamCalibration/ProCamEnsembleCalibration/ProCamEnsembleCalibrationLib.csproj b/ProCamCalibration/ProCamEnsembleCalibration/ProCamEnsembleCalibrationLib.csproj index 10b0743..96c67b0 100644 --- a/ProCamCalibration/ProCamEnsembleCalibration/ProCamEnsembleCalibrationLib.csproj +++ b/ProCamCalibration/ProCamEnsembleCalibration/ProCamEnsembleCalibrationLib.csproj @@ -15,7 +15,9 @@ SAK SAK SAK - e31ed761 + DirectX11_2 + + true @@ -26,7 +28,7 @@ prompt 4 true - true + false pdbonly @@ -45,17 +47,21 @@ ..\packages\Microsoft.Kinect.2.0.1410.19000\lib\net45\Microsoft.Kinect.dll - - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.dll + + False + $(SharpDXPackageBinDir)\SharpDX.dll - - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.D3DCompiler.dll + + False + $(SharpDXPackageBinDir)\SharpDX.Direct2D1.dll - - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct2D1.dll + + False + $(SharpDXPackageBinDir)\SharpDX.Direct3D11.dll - - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct3D11.dll + + False + $(SharpDXPackageBinDir)\SharpDX.DXGI.dll diff --git a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj index b7c5f1b..edbdea1 100644 --- a/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj +++ b/ProCamCalibration/ProjectionMappingSample/ProjectionMappingSample.csproj @@ -5,17 +5,19 @@ Debug AnyCPU {59A35887-D3F7-4AB8-862F-98F68F9F7B51} - Exe + WinExe Properties RoomAliveToolkit ProjectionMappingSample - v4.5 + v4.5.1 512 SAK SAK SAK SAK + DirectX11_2 d0c2a9ee + AnyCPU @@ -27,7 +29,7 @@ prompt 4 true - true + false false @@ -52,19 +54,19 @@ ..\packages\Microsoft.Kinect.2.0.1410.19000\lib\net45\Microsoft.Kinect.dll - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.dll + $(SharpDXPackageBinDir)\SharpDX.dll - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.D3DCompiler.dll + $(SharpDXPackageBinDir)\SharpDX.D3DCompiler.dll - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct2D1.dll + $(SharpDXPackageBinDir)\SharpDX.Direct2D1.dll - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct3D11.dll + $(SharpDXPackageBinDir)\SharpDX.Direct3D11.dll - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.DXGI.dll + $(SharpDXPackageBinDir)\SharpDX.DXGI.dll diff --git a/ProCamCalibration/ProjectorServer/ProjectorServer.cs b/ProCamCalibration/ProjectorServer/ProjectorServer.cs index eebb930..183fbbd 100644 --- a/ProCamCalibration/ProjectorServer/ProjectorServer.cs +++ b/ProCamCalibration/ProjectorServer/ProjectorServer.cs @@ -33,6 +33,7 @@ public void OpenDisplay(int screenIndex) { var projectorForm = new ProjectorForm(screenIndex); projectorForm.Show(); + projectorForm.BringToFront(); projectorForms[screenIndex] = projectorForm; } } @@ -75,6 +76,7 @@ public int NumberOfGrayCodeImages(int screenIndex) public void DisplayGrayCode(int screenIndex, int i) { var projectorForm = projectorForms[screenIndex]; + projectorForm.BringToFront(); projectorForm.DisplayGrayCode(i); } diff --git a/ProCamCalibration/ProjectorServer/ProjectorServer.csproj b/ProCamCalibration/ProjectorServer/ProjectorServer.csproj index 60d9c4f..51c7773 100644 --- a/ProCamCalibration/ProjectorServer/ProjectorServer.csproj +++ b/ProCamCalibration/ProjectorServer/ProjectorServer.csproj @@ -16,6 +16,7 @@ SAK SAK + DirectX11_2 0eef0576 @@ -27,7 +28,7 @@ DEBUG;TRACE prompt 4 - true + false false @@ -69,15 +70,15 @@ False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.dll + $(SharpDXPackageBinDir)\SharpDX.dll False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.Direct2D1.dll + $(SharpDXPackageBinDir)\SharpDX.Direct2D1.dll False - ..\packages\SharpDX.2.6.3\Bin\DirectX11_2-net40\SharpDX.DXGI.dll + $(SharpDXPackageBinDir)\SharpDX.DXGI.dll diff --git a/ProCamCalibration/Shaders/Shaders.vcxproj b/ProCamCalibration/Shaders/Shaders.vcxproj index d268a35..d9f3722 100644 --- a/ProCamCalibration/Shaders/Shaders.vcxproj +++ b/ProCamCalibration/Shaders/Shaders.vcxproj @@ -37,7 +37,7 @@ Application true - v120 + v140 Unicode From ba382ba4c3a3a4cd26c8e73e422e2e25ecd2cc46 Mon Sep 17 00:00:00 2001 From: Roland Smeenk Date: Tue, 16 Feb 2016 15:10:11 +0100 Subject: [PATCH 5/9] Added frustum class for rendering projector and camera positions. Added extra exception handling to prevent crashes. --- .../CalibrateEnsemble.csproj | 9 + .../CalibrateEnsemble/Frustum.cs | 203 ++++++++++++++++++ .../CalibrateEnsemble/MainForm.cs | 149 +++++++++---- .../ProjectionMappingSample/MeshShader.cs | 1 - ProCamCalibration/Shaders/Shaders.vcxproj | 12 ++ .../Shaders/Shaders.vcxproj.filters | 6 + ProCamCalibration/Shaders/frustumPS.hlsl | 14 ++ ProCamCalibration/Shaders/frustumVS.hlsl | 24 +++ 8 files changed, 379 insertions(+), 39 deletions(-) create mode 100644 ProCamCalibration/CalibrateEnsemble/Frustum.cs create mode 100644 ProCamCalibration/Shaders/frustumPS.hlsl create mode 100644 ProCamCalibration/Shaders/frustumVS.hlsl diff --git a/ProCamCalibration/CalibrateEnsemble/CalibrateEnsemble.csproj b/ProCamCalibration/CalibrateEnsemble/CalibrateEnsemble.csproj index 17d25da..00e341b 100644 --- a/ProCamCalibration/CalibrateEnsemble/CalibrateEnsemble.csproj +++ b/ProCamCalibration/CalibrateEnsemble/CalibrateEnsemble.csproj @@ -110,6 +110,7 @@ ConsoleTextBox.cs + Form @@ -155,6 +156,14 @@ Content\DepthAndColorVS.cso PreserveNewest + + Content\frustumPS.cso + PreserveNewest + + + Content\frustumVS.cso + PreserveNewest + diff --git a/ProCamCalibration/CalibrateEnsemble/Frustum.cs b/ProCamCalibration/CalibrateEnsemble/Frustum.cs new file mode 100644 index 0000000..4b87618 --- /dev/null +++ b/ProCamCalibration/CalibrateEnsemble/Frustum.cs @@ -0,0 +1,203 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace RoomAliveToolkit +{ + using System; + using System.IO; + using System.Collections.Generic; + using System.Runtime.InteropServices; + using SharpDX; + using SharpDX.Direct3D11; + using SharpDX.DXGI; + using SharpDX.D3DCompiler; + using SharpDX.WIC; + using Device = SharpDX.Direct3D11.Device; + + public class FrustumShader + { + struct VertexPosition + { + public SharpDX.Vector4 position; + static public int SizeInBytes { get { return 4 * 4; } } + } + + + public SharpDX.Direct3D11.Buffer vertexBuffer; + public VertexBufferBinding vertexBufferBinding; + + public FrustumShader(Device device) + { + // create single vertex buffer + var stream = new DataStream(6 * VertexPosition.SizeInBytes, true, true); + stream.Write(new Vector4(0, 0, 0, 1)); + stream.Write(new Vector4(0, 0, 2, 1)); + stream.Write(new Vector4(0, -0.1f, 0, 1)); + stream.Write(new Vector4(0, 0.1f, 0, 1)); + stream.Write(new Vector4(-0.1f, 0, 0, 1)); + stream.Write(new Vector4(0.1f, 0, 0, 1)); + + stream.Position = 0; + + var vertexBufferDesc = new BufferDescription() + { + BindFlags = BindFlags.VertexBuffer, + CpuAccessFlags = CpuAccessFlags.None, + Usage = ResourceUsage.Default, + SizeInBytes = 6 * VertexPosition.SizeInBytes, + }; + vertexBuffer = new SharpDX.Direct3D11.Buffer(device, stream, vertexBufferDesc); + + stream.Dispose(); + + vertexBufferBinding = new VertexBufferBinding(vertexBuffer, VertexPosition.SizeInBytes, 0); + + + shaderByteCode = new ShaderBytecode(File.ReadAllBytes("Content/frustumVS.cso")); + frustumVS = new VertexShader(device, shaderByteCode); + frustumPS = new PixelShader(device, new ShaderBytecode(File.ReadAllBytes("Content/frustumPS.cso"))); + + // depth stencil state + var depthStencilStateDesc = new DepthStencilStateDescription() + { + IsDepthEnabled = true, + DepthWriteMask = DepthWriteMask.All, + DepthComparison = Comparison.LessEqual, + IsStencilEnabled = false, + }; + depthStencilState = new DepthStencilState(device, depthStencilStateDesc); + + // rasterizer state + var rasterizerStateDesc = new RasterizerStateDescription() + { + CullMode = CullMode.None, + FillMode = FillMode.Wireframe, + IsDepthClipEnabled = true, + IsFrontCounterClockwise = true, + IsMultisampleEnabled = true, + }; + rasterizerState = new RasterizerState(device, rasterizerStateDesc); + + // constant buffer + var VSConstantBufferDesc = new BufferDescription() + { + Usage = ResourceUsage.Dynamic, + BindFlags = BindFlags.ConstantBuffer, + SizeInBytes = VSConstantBuffer.size, + CpuAccessFlags = CpuAccessFlags.Write, + StructureByteStride = 0, + OptionFlags = 0, + }; + vertexShaderConstantBuffer = new SharpDX.Direct3D11.Buffer(device, VSConstantBufferDesc); + + // Pixel shader constant buffer + var PSConstantBufferDesc = new BufferDescription() + { + Usage = ResourceUsage.Dynamic, + BindFlags = BindFlags.ConstantBuffer, + SizeInBytes = PSConstantBuffer.size, + CpuAccessFlags = CpuAccessFlags.Write, + StructureByteStride = 0, + OptionFlags = 0, + }; + pixelShaderConstantBuffer = new SharpDX.Direct3D11.Buffer(device, PSConstantBufferDesc); + + vertexInputLayout = new InputLayout(device, shaderByteCode.Data, new[] + { + new InputElement("SV_POSITION", 0, Format.R32G32B32A32_Float, 0, 0), + }); + } + + // protip: compile shader with /Fc; output gives exact layout + // hlsl matrices are stored column major + // variables are stored on 4-component boundaries; inc. matrix columns + // size is a multiple of 16 + [StructLayout(LayoutKind.Explicit, Size = VSConstantBuffer.size)] + unsafe struct VSConstantBuffer + { + public const int size = 64 + 64 + 64; + + [FieldOffset(0)] + public fixed float world[16]; + [FieldOffset(64)] + public fixed float viewProjection[16]; + [FieldOffset(128)] + public fixed float frustumProjection[16]; + }; + + + [StructLayout(LayoutKind.Explicit, Size = PSConstantBuffer.size)] + unsafe struct PSConstantBuffer + { + public const int size = 16; + + [FieldOffset(0)] + public fixed float color[3]; + }; + + public unsafe void SetVertexShaderConstants(DeviceContext deviceContext, SharpDX.Matrix world, SharpDX.Matrix viewProjection, SharpDX.Matrix frustumProjection) + { + // hlsl matrices are default column order + var constants = new VSConstantBuffer(); + for (int i = 0, col = 0; col < 4; col++) + { + for (int row = 0; row < 4; row++) + { + constants.world[i] = world[row, col]; + constants.viewProjection[i] = viewProjection[row, col]; + constants.frustumProjection[i] = frustumProjection[row, col]; + i++; + } + } + + DataStream dataStream; + deviceContext.MapSubresource(vertexShaderConstantBuffer, MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); + dataStream.Write(constants); + deviceContext.UnmapSubresource(vertexShaderConstantBuffer, 0); + } + + public unsafe void SetPixelShaderConstants(DeviceContext deviceContext, SharpDX.Color3 color) + { + var constants = new PSConstantBuffer(); + for (int i = 0; i < 3; i++) + { + constants.color[i] = color[i]; + } + + DataStream dataStream; + deviceContext.MapSubresource(pixelShaderConstantBuffer, MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); + dataStream.Write(constants); + deviceContext.UnmapSubresource(pixelShaderConstantBuffer, 0); + } + + public void Render(DeviceContext deviceContext) + { + deviceContext.InputAssembler.InputLayout = vertexInputLayout; + deviceContext.InputAssembler.PrimitiveTopology = SharpDX.Direct3D.PrimitiveTopology.LineList; + deviceContext.InputAssembler.SetVertexBuffers(0, vertexBufferBinding); + deviceContext.Rasterizer.State = rasterizerState; + + deviceContext.VertexShader.Set(frustumVS); + deviceContext.VertexShader.SetConstantBuffer(0, vertexShaderConstantBuffer); + + deviceContext.GeometryShader.Set(null); + + deviceContext.PixelShader.SetConstantBuffer(0, pixelShaderConstantBuffer); + deviceContext.PixelShader.Set(frustumPS); + + deviceContext.Draw(6, 0); + } + + VertexShader frustumVS; + PixelShader frustumPS; + ShaderBytecode shaderByteCode; + DepthStencilState depthStencilState; + RasterizerState rasterizerState; + SharpDX.Direct3D11.Buffer vertexShaderConstantBuffer; + SharpDX.Direct3D11.Buffer pixelShaderConstantBuffer; + InputLayout vertexInputLayout; + } +} diff --git a/ProCamCalibration/CalibrateEnsemble/MainForm.cs b/ProCamCalibration/CalibrateEnsemble/MainForm.cs index 6c48f32..58389de 100644 --- a/ProCamCalibration/CalibrateEnsemble/MainForm.cs +++ b/ProCamCalibration/CalibrateEnsemble/MainForm.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.IO; using System.Runtime.InteropServices; +using System.ServiceModel; using System.ServiceModel.Discovery; using System.Windows.Forms; @@ -28,6 +29,7 @@ public MainForm(string[] args) this.args = args; } + FrustumShader frustumShader; ProjectorCameraEnsemble ensemble; string path, directory; string[] args; @@ -77,6 +79,9 @@ protected override void OnLoad(EventArgs e) // viewport viewport = new Viewport(0, 0, videoPanel1.Width, videoPanel1.Height, 0f, 1f); + + frustumShader = new FrustumShader(device); + // shaders var shaderByteCode = new ShaderBytecode(File.ReadAllBytes("Content/DepthAndColorVS.cso")); depthAndColorVS2 = new VertexShader(device, shaderByteCode); @@ -430,31 +435,39 @@ void ColorCameraLoop() { while (live) { - var encodedColorData = camera.Client.LatestJPEGImage(); - - // decode JPEG - var memoryStream = new MemoryStream(encodedColorData); - var stream = new WICStream(imagingFactory, memoryStream); - // decodes to 24 bit BGR - var decoder = new SharpDX.WIC.BitmapDecoder(imagingFactory, stream, SharpDX.WIC.DecodeOptions.CacheOnLoad); - var bitmapFrameDecode = decoder.GetFrame(0); - - // convert to 32 bpp - var formatConverter = new FormatConverter(imagingFactory); - formatConverter.Initialize(bitmapFrameDecode, SharpDX.WIC.PixelFormat.Format32bppBGR); - formatConverter.CopyPixels(nextColorData, Kinect2Calibration.colorImageWidth * 4); // TODO: consider copying directly to texture native memory - //lock (colorData) - // Swap(ref colorData, ref nextColorData); - lock (renderLock) // necessary? + try + { + var encodedColorData = camera.Client.LatestJPEGImage(); + + // decode JPEG + var memoryStream = new MemoryStream(encodedColorData); + var stream = new WICStream(imagingFactory, memoryStream); + // decodes to 24 bit BGR + var decoder = new SharpDX.WIC.BitmapDecoder(imagingFactory, stream, SharpDX.WIC.DecodeOptions.CacheOnLoad); + var bitmapFrameDecode = decoder.GetFrame(0); + + // convert to 32 bpp + var formatConverter = new FormatConverter(imagingFactory); + formatConverter.Initialize(bitmapFrameDecode, SharpDX.WIC.PixelFormat.Format32bppBGR); + formatConverter.CopyPixels(nextColorData, Kinect2Calibration.colorImageWidth * 4); // TODO: consider copying directly to texture native memory + //lock (colorData) + // Swap(ref colorData, ref nextColorData); + lock (renderLock) // necessary? + { + UpdateColorImage(device.ImmediateContext, nextColorData); + } + memoryStream.Close(); + memoryStream.Dispose(); + stream.Dispose(); + decoder.Dispose(); + formatConverter.Dispose(); + bitmapFrameDecode.Dispose(); + } + catch (EndpointNotFoundException) { - UpdateColorImage(device.ImmediateContext, nextColorData); + Console.WriteLine("Could not find Kinect server for live color update."); + live = false; } - memoryStream.Close(); - memoryStream.Dispose(); - stream.Dispose(); - decoder.Dispose(); - formatConverter.Dispose(); - bitmapFrameDecode.Dispose(); } } @@ -464,12 +477,21 @@ void DepthCameraLoop() { while (live) { - nextDepthData = camera.Client.LatestDepthImage(); - //lock (remoteDepthData) - // Swap(ref remoteDepthData, ref nextRemoteDepthData); - lock (renderLock) + try + { + nextDepthData = camera.Client.LatestDepthImage(); + + //lock (remoteDepthData) + // Swap(ref remoteDepthData, ref nextRemoteDepthData); + lock (renderLock) + { + UpdateDepthImage(device.ImmediateContext, nextDepthData); + } + } + catch (EndpointNotFoundException) { - UpdateDepthImage(device.ImmediateContext, nextDepthData); + Console.WriteLine("Could not find Kinect server for live depth update."); + live = false; } } } @@ -561,9 +583,13 @@ void RenderLoop() // render all cameras if (ensemble != null) + { + var viewProjection = view * projection; + foreach (var camera in ensemble.cameras) { if (cameraDeviceResources.ContainsKey(camera)) + { if (cameraDeviceResources[camera].renderEnabled && (camera.pose != null)) { var world = new SharpDX.Matrix(); @@ -573,12 +599,50 @@ void RenderLoop() world.Transpose(); // view and projection matrix are post-multiply - var worldViewProjection = world * view * projection; + var worldViewProjection = world * viewProjection; - SetConstants(deviceContext, camera.calibration, worldViewProjection); + SetConstants(deviceContext, camera.calibration, viewProjection); cameraDeviceResources[camera].Render(deviceContext); } + } + } + + if (renderCameraFrustums) + { + frustumShader.SetPixelShaderConstants(deviceContext, new Color3(1f, 1f, 0f)); + foreach (var camera in ensemble.cameras) + { + SharpDX.Matrix cameraProjection = GetCameraProjectionMatrix(camera); + + var world = new SharpDX.Matrix(); + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + world[i, j] = (float)camera.pose[i, j]; + world.Transpose(); + + frustumShader.SetVertexShaderConstants(deviceContext, world, viewProjection, cameraProjection); + frustumShader.Render(deviceContext); + } + } + + if (renderProjectorFrustums) + { + frustumShader.SetPixelShaderConstants(deviceContext, new Color3(1f, 0f, 1f)); + foreach (var projector in ensemble.projectors) + { + SharpDX.Matrix projectorProjection = GetProjectorProjectionMatrix(projector); + + var world = new SharpDX.Matrix(); + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + world[i, j] = (float)projector.pose[i, j]; + world.Transpose(); + + frustumShader.SetVertexShaderConstants(deviceContext, world, viewProjection, projectorProjection); + frustumShader.Render(deviceContext); + } } + } swapChain.Present(1, PresentFlags.None); } @@ -659,7 +723,13 @@ void renderMenuItem_Click(object sender, EventArgs e) var toolStripMenuItem = (ToolStripMenuItem)sender; var camera = (ProjectorCameraEnsemble.Camera)toolStripMenuItem.Tag; toolStripMenuItem.Checked = !toolStripMenuItem.Checked; - cameraDeviceResources[camera].renderEnabled = toolStripMenuItem.Checked; + + // Prevents KeyNotFoundException if camera was not calibrated and therefore CameraDeviceResource was not created + CameraDeviceResource cameraDeviceResource; + if (cameraDeviceResources.TryGetValue(camera, out cameraDeviceResource)) + { + cameraDeviceResource.renderEnabled = toolStripMenuItem.Checked; + } } bool perspectiveView; @@ -682,7 +752,8 @@ void viewMenuItem_Click(object sender, EventArgs e) toolStripMenuItem.Checked = true; var projector = (ProjectorCameraEnsemble.Projector)toolStripMenuItem.Tag; - SetViewProjectionFromProjector(projector); + projection = GetProjectorProjectionMatrix(projector); + projection.Transpose(); manipulator.View = view; manipulator.OriginalView = view; perspectiveView = false; @@ -1228,7 +1299,7 @@ void AcquireDepthAndColor() // could be method on Projector: - void SetViewProjectionFromProjector(ProjectorCameraEnsemble.Projector projector) + SharpDX.Matrix GetProjectorProjectionMatrix(ProjectorCameraEnsemble.Projector projector) { if ((projector.pose == null) || (projector.cameraMatrix == null)) Console.WriteLine("Projector pose/camera matrix not set. Please perform a calibration."); @@ -1254,12 +1325,13 @@ void SetViewProjectionFromProjector(ProjectorCameraEnsemble.Projector projector) float w = projector.width; float h = projector.height; - projection = GraphicsTransforms.ProjectionMatrixFromCameraMatrix(fx, fy, cx, cy, w, h, near, far); - projection.Transpose(); + return GraphicsTransforms.ProjectionMatrixFromCameraMatrix(fx, fy, cx, cy, w, h, near, far); } + + return SharpDX.Matrix.Identity; } - void SetViewProjectionFromCamera(ProjectorCameraEnsemble.Camera camera) + SharpDX.Matrix GetCameraProjectionMatrix(ProjectorCameraEnsemble.Camera camera) { if ((camera.pose == null) || (camera.calibration.colorCameraMatrix == null)) Console.WriteLine("Camera pose not set."); @@ -1284,9 +1356,10 @@ void SetViewProjectionFromCamera(ProjectorCameraEnsemble.Camera camera) float w = Kinect2Calibration.colorImageWidth; float h = Kinect2Calibration.colorImageHeight; - projection = GraphicsTransforms.ProjectionMatrixFromCameraMatrix(fx, fy, cx, cy, w, h, near, far); - projection.Transpose(); + return GraphicsTransforms.ProjectionMatrixFromCameraMatrix(fx, fy, cx, cy, w, h, near, far); } + + return SharpDX.Matrix.Identity; } void SetDefaultView() diff --git a/ProCamCalibration/ProjectionMappingSample/MeshShader.cs b/ProCamCalibration/ProjectionMappingSample/MeshShader.cs index e0f51f0..b1fe305 100644 --- a/ProCamCalibration/ProjectionMappingSample/MeshShader.cs +++ b/ProCamCalibration/ProjectionMappingSample/MeshShader.cs @@ -289,7 +289,6 @@ public void Render(DeviceContext deviceContext, MeshDeviceResources meshDeviceRe deviceContext.VertexShader.Set(meshVS); deviceContext.VertexShader.SetConstantBuffer(0, vertexShaderConstantBuffer); deviceContext.GeometryShader.Set(null); - deviceContext.PixelShader.Set(meshPS); deviceContext.PixelShader.SetSampler(0, colorSamplerState); deviceContext.PixelShader.SetConstantBuffer(0, pixelShaderConstantBuffer); deviceContext.OutputMerger.SetTargets(depthStencilView, renderTargetView); diff --git a/ProCamCalibration/Shaders/Shaders.vcxproj b/ProCamCalibration/Shaders/Shaders.vcxproj index d9f3722..ab3f2bb 100644 --- a/ProCamCalibration/Shaders/Shaders.vcxproj +++ b/ProCamCalibration/Shaders/Shaders.vcxproj @@ -210,6 +210,18 @@ 5.0 Pixel + + Pixel + Pixel + Pixel + Pixel + + + Vertex + Vertex + Vertex + Vertex + Vertex 5.0 diff --git a/ProCamCalibration/Shaders/Shaders.vcxproj.filters b/ProCamCalibration/Shaders/Shaders.vcxproj.filters index 58be562..ea4aaa7 100644 --- a/ProCamCalibration/Shaders/Shaders.vcxproj.filters +++ b/ProCamCalibration/Shaders/Shaders.vcxproj.filters @@ -46,5 +46,11 @@ Source Files + + Source Files + + + Source Files + \ No newline at end of file diff --git a/ProCamCalibration/Shaders/frustumPS.hlsl b/ProCamCalibration/Shaders/frustumPS.hlsl new file mode 100644 index 0000000..bd960a5 --- /dev/null +++ b/ProCamCalibration/Shaders/frustumPS.hlsl @@ -0,0 +1,14 @@ +struct PSInput +{ + float4 pos : SV_POSITION; +}; + +cbuffer constants : register(b0) +{ + float3 color; +} + +float4 main(PSInput input) : SV_TARGET0 +{ + return float4(color, 1.0); +} diff --git a/ProCamCalibration/Shaders/frustumVS.hlsl b/ProCamCalibration/Shaders/frustumVS.hlsl new file mode 100644 index 0000000..98d68d6 --- /dev/null +++ b/ProCamCalibration/Shaders/frustumVS.hlsl @@ -0,0 +1,24 @@ +struct VSInput +{ + float4 pos : SV_POSITION; +}; + +cbuffer constants : register(b0) +{ + matrix world; + matrix viewProjection; + matrix frustumProjection; +} + +struct VSOutput +{ + float4 pos : SV_POSITION; +}; + +VSOutput main(VSInput input) +{ + VSOutput output = (VSOutput)0; + float4 worldPos = mul(input.pos, world); + output.pos = mul(worldPos, viewProjection); + return output; +} \ No newline at end of file From 5af691d916c04ed09aceedb46839dbbc990d01cc Mon Sep 17 00:00:00 2001 From: Roland Smeenk Date: Tue, 16 Feb 2016 16:04:58 +0100 Subject: [PATCH 6/9] Projector, Kinect Color and Kinect Depth frustums rendered as wireframe. Added Kinect depth camera frustum rendering. --- .../CalibrateEnsemble/Frustum.cs | 41 ++++-- .../CalibrateEnsemble/MainForm.Designer.cs | 26 ++-- .../CalibrateEnsemble/MainForm.cs | 121 ++++++++++++++---- ProCamCalibration/Shaders/frustumVS.hlsl | 4 +- 4 files changed, 149 insertions(+), 43 deletions(-) diff --git a/ProCamCalibration/CalibrateEnsemble/Frustum.cs b/ProCamCalibration/CalibrateEnsemble/Frustum.cs index 4b87618..b59f450 100644 --- a/ProCamCalibration/CalibrateEnsemble/Frustum.cs +++ b/ProCamCalibration/CalibrateEnsemble/Frustum.cs @@ -19,6 +19,8 @@ namespace RoomAliveToolkit public class FrustumShader { + float maxZ = 0.99f; + struct VertexPosition { public SharpDX.Vector4 position; @@ -32,14 +34,33 @@ struct VertexPosition public FrustumShader(Device device) { // create single vertex buffer - var stream = new DataStream(6 * VertexPosition.SizeInBytes, true, true); - stream.Write(new Vector4(0, 0, 0, 1)); - stream.Write(new Vector4(0, 0, 2, 1)); - stream.Write(new Vector4(0, -0.1f, 0, 1)); - stream.Write(new Vector4(0, 0.1f, 0, 1)); - stream.Write(new Vector4(-0.1f, 0, 0, 1)); - stream.Write(new Vector4(0.1f, 0, 0, 1)); - + var stream = new DataStream(24 * VertexPosition.SizeInBytes, true, true); + stream.Write(new Vector4(-1, -1, 0, 1)); + stream.Write(new Vector4( 1, -1, 0, 1)); + stream.Write(new Vector4( 1, -1, 0, 1)); + stream.Write(new Vector4( 1, 1, 0, 1)); + stream.Write(new Vector4( 1, 1, 0, 1)); + stream.Write(new Vector4(-1, 1, 0, 1)); + stream.Write(new Vector4(-1, 1, 0, 1)); + stream.Write(new Vector4(-1, -1, 0, 1)); + + stream.Write(new Vector4(-1, -1, maxZ, 1)); + stream.Write(new Vector4( 1, -1, maxZ, 1)); + stream.Write(new Vector4( 1, -1, maxZ, 1)); + stream.Write(new Vector4( 1, 1, maxZ, 1)); + stream.Write(new Vector4( 1, 1, maxZ, 1)); + stream.Write(new Vector4(-1, 1, maxZ, 1)); + stream.Write(new Vector4(-1, 1, maxZ, 1)); + stream.Write(new Vector4(-1, -1, maxZ, 1)); + + stream.Write(new Vector4(-1, -1, 0, 1)); + stream.Write(new Vector4(-1, -1, maxZ, 1)); + stream.Write(new Vector4( 1, -1, 0, 1)); + stream.Write(new Vector4( 1, -1, maxZ, 1)); + stream.Write(new Vector4( 1, 1, 0, 1)); + stream.Write(new Vector4( 1, 1, maxZ, 1)); + stream.Write(new Vector4(-1, 1, 0, 1)); + stream.Write(new Vector4(-1, 1, maxZ, 1)); stream.Position = 0; var vertexBufferDesc = new BufferDescription() @@ -47,7 +68,7 @@ public FrustumShader(Device device) BindFlags = BindFlags.VertexBuffer, CpuAccessFlags = CpuAccessFlags.None, Usage = ResourceUsage.Default, - SizeInBytes = 6 * VertexPosition.SizeInBytes, + SizeInBytes = 24 * VertexPosition.SizeInBytes, }; vertexBuffer = new SharpDX.Direct3D11.Buffer(device, stream, vertexBufferDesc); @@ -188,7 +209,7 @@ public void Render(DeviceContext deviceContext) deviceContext.PixelShader.SetConstantBuffer(0, pixelShaderConstantBuffer); deviceContext.PixelShader.Set(frustumPS); - deviceContext.Draw(6, 0); + deviceContext.Draw(24, 0); } VertexShader frustumVS; diff --git a/ProCamCalibration/CalibrateEnsemble/MainForm.Designer.cs b/ProCamCalibration/CalibrateEnsemble/MainForm.Designer.cs index b23e94a..8d87ec2 100644 --- a/ProCamCalibration/CalibrateEnsemble/MainForm.Designer.cs +++ b/ProCamCalibration/CalibrateEnsemble/MainForm.Designer.cs @@ -53,7 +53,8 @@ private void InitializeComponent() this.liveViewToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.toolStripSeparator2 = new System.Windows.Forms.ToolStripSeparator(); this.wireframeToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); - this.cameraFrustumsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + this.cameraColorFrustumsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); + this.cameraDepthFrustumsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.projectorFrustumsToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.viewToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); this.perspectiveAtOriginToolStripMenuItem = new System.Windows.Forms.ToolStripMenuItem(); @@ -243,7 +244,8 @@ private void InitializeComponent() this.renderToolStripMenuItem.DropDownItems.AddRange(new System.Windows.Forms.ToolStripItem[] { this.liveViewToolStripMenuItem, this.wireframeToolStripMenuItem, - this.cameraFrustumsToolStripMenuItem, + this.cameraColorFrustumsToolStripMenuItem, + this.cameraDepthFrustumsToolStripMenuItem, this.projectorFrustumsToolStripMenuItem, this.toolStripSeparator2}); this.renderToolStripMenuItem.Name = "renderToolStripMenuItem"; @@ -269,12 +271,19 @@ private void InitializeComponent() this.wireframeToolStripMenuItem.Text = "Wireframe"; this.wireframeToolStripMenuItem.Click += new System.EventHandler(this.wireframeViewToolStripMenuItem_Click); // - // cameraFrustumsToolStripMenuItem + // cameraColorFrustumsToolStripMenuItem // - this.cameraFrustumsToolStripMenuItem.Name = "cameraFrustumsToolStripMenuItem"; - this.cameraFrustumsToolStripMenuItem.Size = new System.Drawing.Size(112, 20); - this.cameraFrustumsToolStripMenuItem.Text = "Camera Frustums"; - this.cameraFrustumsToolStripMenuItem.Click += new System.EventHandler(this.cameraFrustumToolStripMenuItem_Click); + this.cameraColorFrustumsToolStripMenuItem.Name = "cameraColorFrustumsToolStripMenuItem"; + this.cameraColorFrustumsToolStripMenuItem.Size = new System.Drawing.Size(112, 20); + this.cameraColorFrustumsToolStripMenuItem.Text = "Camera Color Frustums"; + this.cameraColorFrustumsToolStripMenuItem.Click += new System.EventHandler(this.cameraColorFrustumToolStripMenuItem_Click); + // + // cameraDepthFrustumsToolStripMenuItem + // + this.cameraDepthFrustumsToolStripMenuItem.Name = "cameraDepthFrustumsToolStripMenuItem"; + this.cameraDepthFrustumsToolStripMenuItem.Size = new System.Drawing.Size(112, 20); + this.cameraDepthFrustumsToolStripMenuItem.Text = "Camera Depth Frustums"; + this.cameraDepthFrustumsToolStripMenuItem.Click += new System.EventHandler(this.cameraDepthFrustumToolStripMenuItem_Click); // // projectorFrustumsToolStripMenuItem // @@ -430,7 +439,8 @@ private void InitializeComponent() private System.Windows.Forms.ToolStripMenuItem calibrateProjectorGroupsToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem optimizePoseToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem wireframeToolStripMenuItem; - private System.Windows.Forms.ToolStripMenuItem cameraFrustumsToolStripMenuItem; + private System.Windows.Forms.ToolStripMenuItem cameraColorFrustumsToolStripMenuItem; + private System.Windows.Forms.ToolStripMenuItem cameraDepthFrustumsToolStripMenuItem; private System.Windows.Forms.ToolStripMenuItem projectorFrustumsToolStripMenuItem; } } \ No newline at end of file diff --git a/ProCamCalibration/CalibrateEnsemble/MainForm.cs b/ProCamCalibration/CalibrateEnsemble/MainForm.cs index 58389de..88b9aea 100644 --- a/ProCamCalibration/CalibrateEnsemble/MainForm.cs +++ b/ProCamCalibration/CalibrateEnsemble/MainForm.cs @@ -607,39 +607,74 @@ void RenderLoop() } } - if (renderCameraFrustums) + if (renderCameraColorFrustums) { frustumShader.SetPixelShaderConstants(deviceContext, new Color3(1f, 1f, 0f)); foreach (var camera in ensemble.cameras) { - SharpDX.Matrix cameraProjection = GetCameraProjectionMatrix(camera); + if (camera.pose != null) + { + SharpDX.Matrix cameraProjection = GetCameraColorProjectionMatrix(camera); + cameraProjection.Transpose(); + cameraProjection.Invert(); - var world = new SharpDX.Matrix(); - for (int i = 0; i < 4; i++) - for (int j = 0; j < 4; j++) - world[i, j] = (float)camera.pose[i, j]; - world.Transpose(); + var world = new SharpDX.Matrix(); + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + world[i, j] = (float)camera.pose[i, j]; + world.Transpose(); - frustumShader.SetVertexShaderConstants(deviceContext, world, viewProjection, cameraProjection); - frustumShader.Render(deviceContext); + frustumShader.SetVertexShaderConstants(deviceContext, world, viewProjection, cameraProjection); + frustumShader.Render(deviceContext); + } } } + if (renderCameraDepthFrustums) + { + frustumShader.SetPixelShaderConstants(deviceContext, new Color3(0f, 1f, 1f)); + foreach (var camera in ensemble.cameras) + { + if (camera.pose != null) + { + SharpDX.Matrix cameraProjection = GetCameraDepthProjectionMatrix(camera); + cameraProjection.Transpose(); + cameraProjection.Invert(); + + var world = new SharpDX.Matrix(); + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + world[i, j] = (float)camera.pose[i, j]; + world.Transpose(); + + frustumShader.SetVertexShaderConstants(deviceContext, world, viewProjection, cameraProjection); + frustumShader.Render(deviceContext); + } + } + } + + //GetCameraDepthProjectionMatrix + if (renderProjectorFrustums) { frustumShader.SetPixelShaderConstants(deviceContext, new Color3(1f, 0f, 1f)); foreach (var projector in ensemble.projectors) { - SharpDX.Matrix projectorProjection = GetProjectorProjectionMatrix(projector); + if (projector.pose != null) + { + SharpDX.Matrix projectorProjection = GetProjectorProjectionMatrix(projector); + projectorProjection.Transpose(); + projectorProjection.Invert(); - var world = new SharpDX.Matrix(); - for (int i = 0; i < 4; i++) - for (int j = 0; j < 4; j++) - world[i, j] = (float)projector.pose[i, j]; - world.Transpose(); + var world = new SharpDX.Matrix(); + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + world[i, j] = (float)projector.pose[i, j]; + world.Transpose(); - frustumShader.SetVertexShaderConstants(deviceContext, world, viewProjection, projectorProjection); - frustumShader.Render(deviceContext); + frustumShader.SetVertexShaderConstants(deviceContext, world, viewProjection, projectorProjection); + frustumShader.Render(deviceContext); + } } } } @@ -999,18 +1034,25 @@ private void wireframeViewToolStripMenuItem_Click(object sender, EventArgs e) UpdateRasterizerState(); } - bool renderCameraFrustums = false; - private void cameraFrustumToolStripMenuItem_Click(object sender, EventArgs e) + bool renderCameraColorFrustums = false; + private void cameraColorFrustumToolStripMenuItem_Click(object sender, EventArgs e) { - renderCameraFrustums = !renderCameraFrustums; - cameraFrustumsToolStripMenuItem.Checked = renderCameraFrustums; + renderCameraColorFrustums = !renderCameraColorFrustums; + cameraColorFrustumsToolStripMenuItem.Checked = renderCameraColorFrustums; + } + + bool renderCameraDepthFrustums = false; + private void cameraDepthFrustumToolStripMenuItem_Click(object sender, EventArgs e) + { + renderCameraDepthFrustums = !renderCameraDepthFrustums; + cameraDepthFrustumsToolStripMenuItem.Checked = renderCameraDepthFrustums; } bool renderProjectorFrustums = false; private void projectorFrustumToolStripMenuItem_Click(object sender, EventArgs e) { renderProjectorFrustums = !renderProjectorFrustums; - projectorFrustumsToolStripMenuItem.Checked = renderCameraFrustums; + projectorFrustumsToolStripMenuItem.Checked = renderProjectorFrustums; } private void videoPanel1_SizeChanged(object sender, EventArgs e) @@ -1331,9 +1373,9 @@ SharpDX.Matrix GetProjectorProjectionMatrix(ProjectorCameraEnsemble.Projector pr return SharpDX.Matrix.Identity; } - SharpDX.Matrix GetCameraProjectionMatrix(ProjectorCameraEnsemble.Camera camera) + SharpDX.Matrix GetCameraColorProjectionMatrix(ProjectorCameraEnsemble.Camera camera) { - if ((camera.pose == null) || (camera.calibration.colorCameraMatrix == null)) + if ((camera.pose == null) || (camera.calibration == null)) Console.WriteLine("Camera pose not set."); else { @@ -1362,6 +1404,37 @@ SharpDX.Matrix GetCameraProjectionMatrix(ProjectorCameraEnsemble.Camera camera) return SharpDX.Matrix.Identity; } + SharpDX.Matrix GetCameraDepthProjectionMatrix(ProjectorCameraEnsemble.Camera camera) + { + if ((camera.pose == null) || (camera.calibration == null)) + Console.WriteLine("Camera pose not set."); + else + { + view = new SharpDX.Matrix(); + for (int i = 0; i < 4; i++) + for (int j = 0; j < 4; j++) + view[i, j] = (float)camera.pose[i, j]; + view.Invert(); + view.Transpose(); + + var cameraMatrix = camera.calibration.depthCameraMatrix; + float fx = (float)cameraMatrix[0, 0]; + float fy = (float)cameraMatrix[1, 1]; + float cx = (float)cameraMatrix[0, 2]; + float cy = (float)cameraMatrix[1, 2]; + + float near = 0.1f; + float far = 100.0f; + + float w = Kinect2Calibration.depthImageWidth; + float h = Kinect2Calibration.depthImageHeight; + + return GraphicsTransforms.ProjectionMatrixFromCameraMatrix(fx, fy, cx, cy, w, h, near, far); + } + + return SharpDX.Matrix.Identity; + } + void SetDefaultView() { view = SharpDX.Matrix.Identity; diff --git a/ProCamCalibration/Shaders/frustumVS.hlsl b/ProCamCalibration/Shaders/frustumVS.hlsl index 98d68d6..be3d775 100644 --- a/ProCamCalibration/Shaders/frustumVS.hlsl +++ b/ProCamCalibration/Shaders/frustumVS.hlsl @@ -18,7 +18,9 @@ struct VSOutput VSOutput main(VSInput input) { VSOutput output = (VSOutput)0; - float4 worldPos = mul(input.pos, world); + + float4 invPos = mul(input.pos, frustumProjection); + float4 worldPos = mul(invPos, world); output.pos = mul(worldPos, viewProjection); return output; } \ No newline at end of file From d5d690991ee8eb9cf508deab1a5f7545a8c3ac00 Mon Sep 17 00:00:00 2001 From: Roland Smeenk Date: Tue, 16 Feb 2016 16:52:51 +0100 Subject: [PATCH 7/9] Removed unused CameraDeviceResource file that was left after reverting. --- .../CameraDeviceResource.cs | 333 ------------------ 1 file changed, 333 deletions(-) delete mode 100644 ProCamCalibration/ProjectionMappingSample/CameraDeviceResource.cs diff --git a/ProCamCalibration/ProjectionMappingSample/CameraDeviceResource.cs b/ProCamCalibration/ProjectionMappingSample/CameraDeviceResource.cs deleted file mode 100644 index 9a7a140..0000000 --- a/ProCamCalibration/ProjectionMappingSample/CameraDeviceResource.cs +++ /dev/null @@ -1,333 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -using SharpDX; -using SharpDX.Direct3D11; -using SharpDX.DXGI; -using SharpDX.WIC; -using System.IO; - - -namespace RoomAliveToolkit -{ - class CameraDeviceResource : IDisposable - { - // encapsulates d3d resources for a camera - public CameraDeviceResource(SharpDX.Direct3D11.Device device, ProjectorCameraEnsemble.Camera camera, Object renderLock, string directory) - { - this.device = device; - this.camera = camera; - this.renderLock = renderLock; - - // Kinect depth image - var depthImageTextureDesc = new Texture2DDescription() - { - Width = Kinect2Calibration.depthImageWidth, - Height = Kinect2Calibration.depthImageHeight, - MipLevels = 1, - ArraySize = 1, - Format = SharpDX.DXGI.Format.R16_UInt, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Dynamic, - BindFlags = BindFlags.ShaderResource, - CpuAccessFlags = CpuAccessFlags.Write, - }; - depthImageTexture = new Texture2D(device, depthImageTextureDesc); - depthImageTextureRV = new ShaderResourceView(device, depthImageTexture); - - var floatDepthImageTextureDesc = new Texture2DDescription() - { - Width = Kinect2Calibration.depthImageWidth, - Height = Kinect2Calibration.depthImageHeight, - MipLevels = 1, - ArraySize = 1, - Format = SharpDX.DXGI.Format.R32_Float, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Default, - BindFlags = BindFlags.RenderTarget | BindFlags.ShaderResource, - CpuAccessFlags = CpuAccessFlags.None, - }; - - floatDepthImageTexture = new Texture2D(device, floatDepthImageTextureDesc); - floatDepthImageRV = new ShaderResourceView(device, floatDepthImageTexture); - floatDepthImageRenderTargetView = new RenderTargetView(device, floatDepthImageTexture); - - floatDepthImageTexture2 = new Texture2D(device, floatDepthImageTextureDesc); - floatDepthImageRV2 = new ShaderResourceView(device, floatDepthImageTexture2); - floatDepthImageRenderTargetView2 = new RenderTargetView(device, floatDepthImageTexture2); - - // Kinect color image - var colorImageStagingTextureDesc = new Texture2DDescription() - { - Width = Kinect2Calibration.colorImageWidth, - Height = Kinect2Calibration.colorImageHeight, - MipLevels = 1, - ArraySize = 1, - Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Dynamic, - BindFlags = BindFlags.ShaderResource, - CpuAccessFlags = CpuAccessFlags.Write - }; - colorImageStagingTexture = new Texture2D(device, colorImageStagingTextureDesc); - - var colorImageTextureDesc = new Texture2DDescription() - { - Width = Kinect2Calibration.colorImageWidth, - Height = Kinect2Calibration.colorImageHeight, - MipLevels = 0, - ArraySize = 1, - Format = SharpDX.DXGI.Format.B8G8R8A8_UNorm, - SampleDescription = new SharpDX.DXGI.SampleDescription(1, 0), - Usage = ResourceUsage.Default, - BindFlags = BindFlags.ShaderResource | BindFlags.RenderTarget, - CpuAccessFlags = CpuAccessFlags.None, - OptionFlags = ResourceOptionFlags.GenerateMipMaps - }; - colorImageTexture = new Texture2D(device, colorImageTextureDesc); - colorImageTextureRV = new ShaderResourceView(device, colorImageTexture); - - // vertex buffer - var table = camera.calibration.ComputeDepthFrameToCameraSpaceTable(); - int numVertices = 6 * (Kinect2Calibration.depthImageWidth - 1) * (Kinect2Calibration.depthImageHeight - 1); - var vertices = new VertexPosition[numVertices]; - - Int3[] quadOffsets = new Int3[] - { - new Int3(0, 0, 0), - new Int3(1, 0, 0), - new Int3(0, 1, 0), - new Int3(1, 0, 0), - new Int3(1, 1, 0), - new Int3(0, 1, 0), - }; - - int vertexIndex = 0; - for (int y = 0; y < Kinect2Calibration.depthImageHeight - 1; y++) - { - for (int x = 0; x < Kinect2Calibration.depthImageWidth - 1; x++) - { - for (int i = 0; i < 6; i++) - { - int vertexX = x + quadOffsets[i].X; - int vertexY = y + quadOffsets[i].Y; - - var point = table[Kinect2Calibration.depthImageWidth * vertexY + vertexX]; - - var vertex = new VertexPosition(); - vertex.position = new SharpDX.Vector4(point.X, point.Y, vertexX, vertexY); - vertices[vertexIndex++] = vertex; - } - } - } - - var stream = new DataStream(numVertices * VertexPosition.SizeInBytes, true, true); - stream.WriteRange(vertices); - stream.Position = 0; - - var vertexBufferDesc = new BufferDescription() - { - BindFlags = BindFlags.VertexBuffer, - CpuAccessFlags = CpuAccessFlags.None, - Usage = ResourceUsage.Default, - SizeInBytes = numVertices * VertexPosition.SizeInBytes, - }; - vertexBuffer = new SharpDX.Direct3D11.Buffer(device, stream, vertexBufferDesc); - - vertexBufferBinding = new VertexBufferBinding(vertexBuffer, VertexPosition.SizeInBytes, 0); - - stream.Dispose(); - - //var colorImage = new RoomAliveToolkit.ARGBImage(Kinect2Calibration.colorImageWidth, Kinect2Calibration.colorImageHeight); - //ProjectorCameraEnsemble.LoadFromTiff(imagingFactory, colorImage, directory + "/camera" + camera.name + "/color.tiff"); - - var depthImage = new RoomAliveToolkit.ShortImage(Kinect2Calibration.depthImageWidth, Kinect2Calibration.depthImageHeight); - ProjectorCameraEnsemble.LoadFromTiff(imagingFactory, depthImage, directory + "/camera" + camera.name + "/mean.tiff"); - - lock (renderLock) // necessary? - { - //UpdateColorImage(device.ImmediateContext, colorImage.DataIntPtr); - UpdateDepthImage(device.ImmediateContext, depthImage.DataIntPtr); - } - - //colorImage.Dispose(); - depthImage.Dispose(); - } - - struct VertexPosition - { - public SharpDX.Vector4 position; - static public int SizeInBytes { get { return 4 * 4; } } - } - - public void Dispose() - { - depthImageTexture.Dispose(); - depthImageTextureRV.Dispose(); - colorImageTexture.Dispose(); - colorImageTextureRV.Dispose(); - colorImageStagingTexture.Dispose(); - vertexBuffer.Dispose(); - } - - SharpDX.Direct3D11.Device device; - public Texture2D depthImageTexture, floatDepthImageTexture, floatDepthImageTexture2; - public ShaderResourceView depthImageTextureRV, floatDepthImageRV, floatDepthImageRV2; - public RenderTargetView floatDepthImageRenderTargetView, floatDepthImageRenderTargetView2; - public Texture2D colorImageTexture; - public ShaderResourceView colorImageTextureRV; - public Texture2D colorImageStagingTexture; - public SharpDX.Direct3D11.Buffer vertexBuffer; - VertexBufferBinding vertexBufferBinding; - ProjectorCameraEnsemble.Camera camera; - public bool renderEnabled = true; - - public void UpdateDepthImage(DeviceContext deviceContext, IntPtr depthImage) - { - DataStream dataStream; - deviceContext.MapSubresource(depthImageTexture, 0, - MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); - dataStream.WriteRange(depthImage, Kinect2Calibration.depthImageWidth * Kinect2Calibration.depthImageHeight * 2); - deviceContext.UnmapSubresource(depthImageTexture, 0); - } - - public void UpdateDepthImage(DeviceContext deviceContext, byte[] depthImage) - { - DataStream dataStream; - deviceContext.MapSubresource(depthImageTexture, 0, - MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); - dataStream.WriteRange(depthImage, 0, Kinect2Calibration.depthImageWidth * Kinect2Calibration.depthImageHeight * 2); - deviceContext.UnmapSubresource(depthImageTexture, 0); - } - - public void UpdateColorImage(DeviceContext deviceContext, IntPtr colorImage) - { - DataStream dataStream; - deviceContext.MapSubresource(colorImageStagingTexture, 0, - MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); - dataStream.WriteRange(colorImage, Kinect2Calibration.colorImageWidth * Kinect2Calibration.colorImageHeight * 4); - deviceContext.UnmapSubresource(colorImageStagingTexture, 0); - - var resourceRegion = new ResourceRegion() - { - Left = 0, - Top = 0, - Right = Kinect2Calibration.colorImageWidth, - Bottom = Kinect2Calibration.colorImageHeight, - Front = 0, - Back = 1, - }; - deviceContext.CopySubresourceRegion(colorImageStagingTexture, 0, resourceRegion, colorImageTexture, 0); - deviceContext.GenerateMips(colorImageTextureRV); - } - - public void UpdateColorImage(DeviceContext deviceContext, byte[] colorImage) - { - DataStream dataStream; - deviceContext.MapSubresource(colorImageStagingTexture, 0, - MapMode.WriteDiscard, SharpDX.Direct3D11.MapFlags.None, out dataStream); - dataStream.WriteRange(colorImage, 0, Kinect2Calibration.colorImageWidth * Kinect2Calibration.colorImageHeight * 4); - deviceContext.UnmapSubresource(colorImageStagingTexture, 0); - - var resourceRegion = new ResourceRegion() - { - Left = 0, - Top = 0, - Right = Kinect2Calibration.colorImageWidth, - Bottom = Kinect2Calibration.colorImageHeight, - Front = 0, - Back = 1, - }; - deviceContext.CopySubresourceRegion(colorImageStagingTexture, 0, resourceRegion, colorImageTexture, 0); - deviceContext.GenerateMips(colorImageTextureRV); - } - - public void Render(DeviceContext deviceContext) - { - deviceContext.InputAssembler.SetVertexBuffers(0, vertexBufferBinding); - deviceContext.VertexShader.SetShaderResource(0, depthImageTextureRV); - deviceContext.PixelShader.SetShaderResource(0, colorImageTextureRV); - deviceContext.Draw((Kinect2Calibration.depthImageWidth - 1) * (Kinect2Calibration.depthImageHeight - 1) * 6, 0); - } - - bool live = false; - - public void StartLive() - { - live = true; - //new System.Threading.Thread(ColorCameraLoop).Start(); - new System.Threading.Thread(DepthCameraLoop).Start(); - } - - public void StopLive() - { - live = false; - } - - - Object renderLock; - public bool depthImageChanged = true; - - //byte[] colorData = new byte[4 * Kinect2.Kinect2Calibration.colorImageWidth * Kinect2.Kinect2Calibration.colorImageHeight]; - byte[] nextColorData = new byte[4 * RoomAliveToolkit.Kinect2Calibration.colorImageWidth * RoomAliveToolkit.Kinect2Calibration.colorImageHeight]; - SharpDX.WIC.ImagingFactory2 imagingFactory = new SharpDX.WIC.ImagingFactory2(); - void ColorCameraLoop() - { - while (true) - { - var encodedColorData = camera.Client.LatestJPEGImage(); - - // decode JPEG - var memoryStream = new MemoryStream(encodedColorData); - var stream = new WICStream(imagingFactory, memoryStream); - // decodes to 24 bit BGR - var decoder = new SharpDX.WIC.BitmapDecoder(imagingFactory, stream, SharpDX.WIC.DecodeOptions.CacheOnLoad); - var bitmapFrameDecode = decoder.GetFrame(0); - - // convert to 32 bpp - var formatConverter = new FormatConverter(imagingFactory); - formatConverter.Initialize(bitmapFrameDecode, SharpDX.WIC.PixelFormat.Format32bppBGR); - formatConverter.CopyPixels(nextColorData, Kinect2Calibration.colorImageWidth * 4); // TODO: consider copying directly to texture native memory - //lock (colorData) - // Swap(ref colorData, ref nextColorData); - lock (renderLock) // necessary? - { - UpdateColorImage(device.ImmediateContext, nextColorData); - } - memoryStream.Close(); - memoryStream.Dispose(); - stream.Dispose(); - decoder.Dispose(); - formatConverter.Dispose(); - bitmapFrameDecode.Dispose(); - } - } - - //byte[] depthData = new byte[2 * Kinect2.Kinect2Calibration.depthImageWidth * Kinect2.Kinect2Calibration.depthImageHeight]; - byte[] nextDepthData; - void DepthCameraLoop() - { - while (true) - { - nextDepthData = camera.Client.LatestDepthImage(); - //lock (remoteDepthData) - // Swap(ref remoteDepthData, ref nextRemoteDepthData); - lock (renderLock) - { - depthImageChanged = true; - UpdateDepthImage(device.ImmediateContext, nextDepthData); - } - } - } - - static void Swap(ref T first, ref T second) - { - T temp = first; - first = second; - second = temp; - } - } -} From c138ed1928d2ea3d79717d10613cf5e29dc6e59f Mon Sep 17 00:00:00 2001 From: Roland Smeenk Date: Tue, 16 Feb 2016 22:56:34 +0100 Subject: [PATCH 8/9] Fixed missing world matrix for Kinect camera's. Reduced frustum range a little. --- ProCamCalibration/CalibrateEnsemble/Frustum.cs | 2 +- ProCamCalibration/CalibrateEnsemble/MainForm.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ProCamCalibration/CalibrateEnsemble/Frustum.cs b/ProCamCalibration/CalibrateEnsemble/Frustum.cs index b59f450..2933efa 100644 --- a/ProCamCalibration/CalibrateEnsemble/Frustum.cs +++ b/ProCamCalibration/CalibrateEnsemble/Frustum.cs @@ -19,7 +19,7 @@ namespace RoomAliveToolkit public class FrustumShader { - float maxZ = 0.99f; + float maxZ = 0.97f; struct VertexPosition { diff --git a/ProCamCalibration/CalibrateEnsemble/MainForm.cs b/ProCamCalibration/CalibrateEnsemble/MainForm.cs index 88b9aea..73dd4a1 100644 --- a/ProCamCalibration/CalibrateEnsemble/MainForm.cs +++ b/ProCamCalibration/CalibrateEnsemble/MainForm.cs @@ -601,7 +601,7 @@ void RenderLoop() // view and projection matrix are post-multiply var worldViewProjection = world * viewProjection; - SetConstants(deviceContext, camera.calibration, viewProjection); + SetConstants(deviceContext, camera.calibration, worldViewProjection); cameraDeviceResources[camera].Render(deviceContext); } } From f7ac8488a900e82a3550541cee557aada4728ce2 Mon Sep 17 00:00:00 2001 From: Roland Smeenk Date: Tue, 16 Feb 2016 23:12:47 +0100 Subject: [PATCH 9/9] Camera color and depth frustums now switch with camera toggles. Renamed frustumProjection to inverseProjection for clarity. --- .../CalibrateEnsemble/Frustum.cs | 6 +- .../CalibrateEnsemble/MainForm.cs | 58 +++++++++---------- ProCamCalibration/Shaders/frustumVS.hlsl | 4 +- 3 files changed, 31 insertions(+), 37 deletions(-) diff --git a/ProCamCalibration/CalibrateEnsemble/Frustum.cs b/ProCamCalibration/CalibrateEnsemble/Frustum.cs index 2933efa..6b64e18 100644 --- a/ProCamCalibration/CalibrateEnsemble/Frustum.cs +++ b/ProCamCalibration/CalibrateEnsemble/Frustum.cs @@ -146,7 +146,7 @@ unsafe struct VSConstantBuffer [FieldOffset(64)] public fixed float viewProjection[16]; [FieldOffset(128)] - public fixed float frustumProjection[16]; + public fixed float inverseProjection[16]; }; @@ -159,7 +159,7 @@ unsafe struct PSConstantBuffer public fixed float color[3]; }; - public unsafe void SetVertexShaderConstants(DeviceContext deviceContext, SharpDX.Matrix world, SharpDX.Matrix viewProjection, SharpDX.Matrix frustumProjection) + public unsafe void SetVertexShaderConstants(DeviceContext deviceContext, SharpDX.Matrix world, SharpDX.Matrix viewProjection, SharpDX.Matrix inverseProjection) { // hlsl matrices are default column order var constants = new VSConstantBuffer(); @@ -169,7 +169,7 @@ public unsafe void SetVertexShaderConstants(DeviceContext deviceContext, SharpDX { constants.world[i] = world[row, col]; constants.viewProjection[i] = viewProjection[row, col]; - constants.frustumProjection[i] = frustumProjection[row, col]; + constants.inverseProjection[i] = inverseProjection[row, col]; i++; } } diff --git a/ProCamCalibration/CalibrateEnsemble/MainForm.cs b/ProCamCalibration/CalibrateEnsemble/MainForm.cs index 73dd4a1..96061d4 100644 --- a/ProCamCalibration/CalibrateEnsemble/MainForm.cs +++ b/ProCamCalibration/CalibrateEnsemble/MainForm.cs @@ -607,54 +607,48 @@ void RenderLoop() } } - if (renderCameraColorFrustums) + foreach (var camera in ensemble.cameras) { - frustumShader.SetPixelShaderConstants(deviceContext, new Color3(1f, 1f, 0f)); - foreach (var camera in ensemble.cameras) + if (cameraDeviceResources.ContainsKey(camera)) { - if (camera.pose != null) + if (cameraDeviceResources[camera].renderEnabled && (camera.pose != null)) { - SharpDX.Matrix cameraProjection = GetCameraColorProjectionMatrix(camera); - cameraProjection.Transpose(); - cameraProjection.Invert(); - var world = new SharpDX.Matrix(); for (int i = 0; i < 4; i++) for (int j = 0; j < 4; j++) world[i, j] = (float)camera.pose[i, j]; world.Transpose(); - frustumShader.SetVertexShaderConstants(deviceContext, world, viewProjection, cameraProjection); - frustumShader.Render(deviceContext); - } - } - } + // view and projection matrix are post-multiply + var worldViewProjection = world * viewProjection; - if (renderCameraDepthFrustums) - { - frustumShader.SetPixelShaderConstants(deviceContext, new Color3(0f, 1f, 1f)); - foreach (var camera in ensemble.cameras) - { - if (camera.pose != null) - { - SharpDX.Matrix cameraProjection = GetCameraDepthProjectionMatrix(camera); - cameraProjection.Transpose(); - cameraProjection.Invert(); + if (renderCameraColorFrustums) + { + frustumShader.SetPixelShaderConstants(deviceContext, new Color3(1f, 1f, 0f)); - var world = new SharpDX.Matrix(); - for (int i = 0; i < 4; i++) - for (int j = 0; j < 4; j++) - world[i, j] = (float)camera.pose[i, j]; - world.Transpose(); + SharpDX.Matrix cameraProjection = GetCameraColorProjectionMatrix(camera); + cameraProjection.Transpose(); + cameraProjection.Invert(); - frustumShader.SetVertexShaderConstants(deviceContext, world, viewProjection, cameraProjection); - frustumShader.Render(deviceContext); + frustumShader.SetVertexShaderConstants(deviceContext, world, viewProjection, cameraProjection); + frustumShader.Render(deviceContext); + } + + if (renderCameraDepthFrustums) + { + frustumShader.SetPixelShaderConstants(deviceContext, new Color3(0f, 1f, 1f)); + + SharpDX.Matrix cameraProjection = GetCameraDepthProjectionMatrix(camera); + cameraProjection.Transpose(); + cameraProjection.Invert(); + + frustumShader.SetVertexShaderConstants(deviceContext, world, viewProjection, cameraProjection); + frustumShader.Render(deviceContext); + } } } } - //GetCameraDepthProjectionMatrix - if (renderProjectorFrustums) { frustumShader.SetPixelShaderConstants(deviceContext, new Color3(1f, 0f, 1f)); diff --git a/ProCamCalibration/Shaders/frustumVS.hlsl b/ProCamCalibration/Shaders/frustumVS.hlsl index be3d775..6f78e4f 100644 --- a/ProCamCalibration/Shaders/frustumVS.hlsl +++ b/ProCamCalibration/Shaders/frustumVS.hlsl @@ -7,7 +7,7 @@ cbuffer constants : register(b0) { matrix world; matrix viewProjection; - matrix frustumProjection; + matrix inverseProjection; } struct VSOutput @@ -19,7 +19,7 @@ VSOutput main(VSInput input) { VSOutput output = (VSOutput)0; - float4 invPos = mul(input.pos, frustumProjection); + float4 invPos = mul(input.pos, inverseProjection); float4 worldPos = mul(invPos, world); output.pos = mul(worldPos, viewProjection); return output;