kivy/examples/kinect/kinectviewer.py

import freenect
from time import sleep
from threading import Thread
from collections import deque
from kivy.app import App
from kivy.clock import Clock
from kivy.properties import NumericProperty, StringProperty
from kivy.graphics import RenderContext, Color, Rectangle
from kivy.graphics.texture import Texture
from kivy.core.window import Window
from kivy.uix.widget import Widget
from kivy.uix.slider import Slider
from kivy.uix.boxlayout import BoxLayout


fragment_header = '''
#ifdef GL_ES
    precision highp float;
#endif

/* Outputs from the vertex shader */
varying vec4 frag_color;
varying vec2 tex_coord0;

/* uniform texture samplers */
uniform sampler2D texture0;

/* custom input */
uniform float depth_range;
uniform vec2 size;
'''

hsv_func = '''
vec3 HSVtoRGB(vec3 color) {
    float f,p,q,t, hueRound;
    int hueIndex;
    float hue, saturation, v;
    vec3 result;

    /* just for clarity */
    hue = color.r;
    saturation = color.g;
    v = color.b;

    hueRound = floor(hue * 6.0);
    hueIndex = mod(int(hueRound), 6.);
    f = (hue * 6.0) - hueRound;
    p = v * (1.0 - saturation);
    q = v * (1.0 - f*saturation);
    t = v * (1.0 - (1.0 - f)*saturation);

    switch(hueIndex) {
        case 0:
            result = vec3(v,t,p);
        break;
        case 1:
            result = vec3(q,v,p);
        break;
        case 2:
            result = vec3(p,v,t);
        break;
        case 3:
            result = vec3(p,q,v);
        break;
        case 4:
            result = vec3(t,p,v);
        break;
        case 5:
            result = vec3(v,p,q);
        break;
    }
    return result;
}
'''

rgb_kinect = fragment_header + '''
void main (void) {
    float value = texture2D(texture0, tex_coord0).r;
    value = mod(value * depth_range, 1.);
    vec3 col = vec3(0., 0., 0.);
    if ( value <= 0.33 )
        col.r = clamp(value, 0., 0.33) * 3.;
    if ( value <= 0.66 )
        col.g = clamp(value - 0.33, 0., 0.33) * 3.;
    col.b = clamp(value - 0.66, 0., 0.33) * 3.;
    gl_FragColor = vec4(col, 1.);
}
'''

points_kinect = fragment_header + hsv_func + '''
void main (void) {
    // threshold used to reduce the depth (better result)
    const int th = 5;

    // size of a square
    int square = floor(depth_range);

    // number of square on the display
    vec2 count = size / square;

    // current position of the square
    vec2 pos = floor(tex_coord0.xy * count) / count;

    // texture step to pass to another square
    vec2 step = 1 / count;

    // texture step to pass to another pixel
    vec2 pxstep = 1 / size;

    // center of the square
    vec2 center = pos + step / 2.;

    // calculate average of every pixels in the square
    float s = 0, x, y;
    for (x = 0; x < square; x++) {
        for (y = 0; y < square; y++) {
            s += texture2D(texture0, pos + pxstep * vec2(x,y)).r;
        }
    }
    float v = s / (square * square);

    // threshold the value
    float dr = th / 10.;
    v = min(v, dr) / dr;

    // calculate the distance between the center of the square and current
    // pixel; display the pixel only if the distance is inside the circle
    float vdist = length(abs(tex_coord0 - center) * size / square);
    float value = 1 - v;
    if ( vdist < value ) {
        vec3 col = HSVtoRGB(vec3(value, 1., 1.));
        gl_FragColor = vec4(col, 1);
    }
}
'''
hsv_kinect = fragment_header + hsv_func + '''
void main (void) {
    float value = texture2D(texture0, tex_coord0).r;
    value = mod(value * depth_range, 1.);
    vec3 col = HSVtoRGB(vec3(value, 1., 1.));
    gl_FragColor = vec4(col, 1.);
}
'''


class KinectDepth(Thread):

    def __init__(self, *largs, **kwargs):
        super(KinectDepth, self).__init__(*largs, **kwargs)
        self.daemon = True
        self.queue = deque()
        self.quit = False
        self.index = 0

    def run(self):
        q = self.queue
        while not self.quit:
            depths = freenect.sync_get_depth(index=self.index)
            if depths is None:
                sleep(2)
                continue
            q.appendleft(depths)

    def pop(self):
        return self.queue.pop()


class KinectViewer(Widget):

    depth_range = NumericProperty(7.7)

    shader = StringProperty("rgb")

    index = NumericProperty(0)

    def __init__(self, **kwargs):
        # change the default canvas to RenderContext, we can change the shader
        self.canvas = RenderContext()
        self.canvas.shader.fs = hsv_kinect

        # add kinect depth provider, and start the thread
        self.kinect = KinectDepth()
        self.kinect.start()

        # parent init
        super(KinectViewer, self).__init__(**kwargs)

        # allocate texture for pushing depth
        self.texture = Texture.create(
            size=(640, 480), colorfmt='luminance', bufferfmt='ushort')
        self.texture.flip_vertical()

        # create default canvas element
        with self.canvas:
            Color(1, 1, 1)
            Rectangle(size=Window.size, texture=self.texture)

        # add a little clock to update our glsl
        Clock.schedule_interval(self.update_transformation, 0)

    def on_index(self, instance, value):
        self.kinect.index = value

    def on_shader(self, instance, value):
        if value == 'rgb':
            self.canvas.shader.fs = rgb_kinect
        elif value == 'hsv':
            self.canvas.shader.fs = hsv_kinect
        elif value == 'points':
            self.canvas.shader.fs = points_kinect

    def update_transformation(self, *largs):
        # update projection mat and uvsize
        self.canvas['projection_mat'] = Window.render_context['projection_mat']
        self.canvas['depth_range'] = self.depth_range
        self.canvas['size'] = list(map(float, self.size))
        try:
            value = self.kinect.pop()
        except:
            return
        f = value[0].astype('ushort') * 32
        self.texture.blit_buffer(
            f.tostring(), colorfmt='luminance', bufferfmt='ushort')
        self.canvas.ask_update()


class KinectViewerApp(App):

    def build(self):
        root = BoxLayout(orientation='vertical')

        self.viewer = viewer = KinectViewer(
            index=self.config.getint('kinect', 'index'),
            shader=self.config.get('shader', 'theme'))
        root.add_widget(viewer)

        toolbar = BoxLayout(size_hint=(1, None), height=50)
        root.add_widget(toolbar)

        slider = Slider(min=1., max=32., value=1.)

        def update_depth_range(instance, value):
            viewer.depth_range = value

        slider.bind(value=update_depth_range)
        toolbar.add_widget(slider)

        return root

    def build_config(self, config):
        config.add_section('kinect')
        config.set('kinect', 'index', '0')
        config.add_section('shader')
        config.set('shader', 'theme', 'rgb')

    def build_settings(self, settings):
        settings.add_json_panel('Kinect Viewer', self.config, data='''[
            { "type": "title", "title": "Kinect" },
            { "type": "numeric", "title": "Index",
              "desc": "Kinect index, from 0 to X",
              "section": "kinect", "key": "index" },
            { "type": "title", "title": "Shaders" },
            { "type": "options", "title": "Theme",
              "desc": "Shader to use for a specific visualization",
              "section": "shader", "key": "theme",
              "options": ["rgb", "hsv", "points"]}
        ]''')

    def on_config_change(self, config, section, key, value):
        if config is not self.config:
            return
        token = (section, key)
        if token == ('kinect', 'index'):
            self.viewer.index = int(value)
        elif token == ('shader', 'theme'):
            if value == 'rgb':
                self.viewer.canvas.shader.fs = rgb_kinect
            elif value == 'hsv':
                self.viewer.shader = value


if __name__ == '__main__':
    KinectViewerApp().run()