Back to posts.

Real-time oil painting with openGL

I'm testing different approaches towards a real-time 1080p oil painting simulation. To create an oil painting, I capture the scene into a texture (using FBO RTT) and use a shader to calculate the gradients (from grayscale values). I use these gradients to align quads in the same direction so they align nicely with the color grades in the source image. I write all the gradients into a GL_R32F float texture, for each pixel. (this is all work in progress).

Gradient vectors on GPU:

Then in a next step, I render N-quads where I omit vertex attributes and a VBO completely by using the gl_VertexID keyword to emit the correct vertex for the quad/triangle strip (see below for source). This is a lot faster then generating vertices in a geometry shader!

Aligned quads. Input from slit-scan:

During this second step I rotate the quad based on the angle/gradient I calculated during the first pass (from the u_angle_tex uniform). I also get the correct from the scene texture.

With my radeon 4850 I'm not yet getting a satisfying framerate: 10-15fps with 100.000 quads. Though with my GTX 670 I'm getting 60fps with 3.000.000 quads (with texturing!)

Some more screenshots of fake oil painting strokes:

#ifndef ROXLU_OILS_H
#define ROXLU_OILS_H
 
#include <glr/GL.h>
#include <roxlu/math/Random.h>
#include <roxlu/math/Vec2.h>
#include <roxlu/opengl/GL.h>
#include <roxlu/opengl/Error.h>
 
using namespace roxlu;
using namespace gl;
 
static const char* OSA_VS = ""
  "#version 150\n"
  "in vec4 a_pos;"
 
  "void main() {"
  "  gl_Position = a_pos; "
  "}";
 
static const char* OSA_FS = ""
  "#version 150\n"
  "uniform sampler2D u_scene;"
  "out float angle;"
 
  "float bw(vec3 col) {"
  "  return (col.r * 0.2126 + col.g * 0.7152 + col.b * 0.0722); "
  "}"
 
  "void main() {"
  "  ivec2 tc = ivec2(gl_FragCoord);"
  "  vec3 col_left = texelFetchOffset(u_scene, tc, 0, ivec2(-1, 0)).rgb;"
  "  vec3 col_right = texelFetchOffset(u_scene, tc, 0, ivec2(1, 0)).rgb;"
  "  vec3 col_top = texelFetchOffset(u_scene, tc, 0, ivec2(0, -1)).rgb;"
  "  vec3 col_bottom = texelFetchOffset(u_scene, tc, 0, ivec2(0, 1)).rgb;"
  "  float left = bw(col_left); "
  "  float right = bw(col_right); "
  "  float top = bw(col_top); "
  "  float bottom = bw(col_bottom); "
  "  float dx = (right - left) * 0.5;"
  "  float dy = (bottom - top) * 0.5;"
  "  angle = atan(dy, dx); "
  "}";
 
 
static const char* OS_VS = ""
  "#version 150\n"
  "uniform mat4 u_pm;"
  "uniform samplerBuffer u_pos_buffer;"
  "uniform sampler2D u_angle_tex;"
  "uniform sampler2D u_scene_tex;"
 
  "const float w = 3.1;"
  "const float h = 12.0;"
  "out vec3 v_col;"
 
  "const vec2 data[4] = vec2[] ("
  "     vec2(-w, h), "
  "     vec2(-w, -h), "
  "     vec2( w,  h), "
  "     vec2( w, -h) "
  ");"
 
  "void main() {"
 
  "  vec2 pos = texelFetch(u_pos_buffer, gl_InstanceID).rg; "
  "  float angle = texelFetch(u_angle_tex, ivec2(720 - pos.x, 1280 - pos.y), 0).r; "
  "  float ca = cos(angle);"
  "  float sa = sin(angle);"
 
  "  mat4 rot_z; "
  "  rot_z[0].x = ca; "
  "  rot_z[0].y = -sa; "
  "  rot_z[0].z = 0.0; "
  "  rot_z[0].w = 0.0; "
 
  "  rot_z[1].x = sa; "
  "  rot_z[1].y = ca; "
  "  rot_z[1].z = 0.0; "
  "  rot_z[1].w = 0.0; "
 
  "  rot_z[2].x = 0.0; "
  "  rot_z[2].y = 0.0; "
  "  rot_z[2].z = 1.0; "
  "  rot_z[2].w = 0.0; "
 
  "  rot_z[3].x = 0.0; "
  "  rot_z[3].y = 0.0; "
  "  rot_z[3].z = 0.0; "
  "  rot_z[3].w = 1.0; "
 
  "  vec4 rot_vert = vec4(data[gl_VertexID], 0.0, 0.0) * rot_z;"
  "  vec4 vpos = rot_vert + vec4(pos, 0.0, 1.0);"
  "  gl_Position = u_pm * ( vpos ); "
  "  v_col = texelFetch(u_scene_tex, ivec2(720 - pos.x, 1280 - pos.y), 0).rgb; "
  "}"
  "";
 
static const char* OS_FS = ""
  "#version 150\n"
  "out vec4 fragcolor;"
  "in vec3 v_col;"
 
  "void main() {"
  "  fragcolor = vec4(v_col, 1.0); "
  "}"
  "";
 
class Oils {
 public:
  Oils();
  ~Oils();
  bool setup(int winW, int winH);
  void draw();
  void generateBrushVertices();
  void blitScene();
  void findAngles();
 
 public:
  int win_w;
  int win_h;
 
  /* angle */
  GLuint fs_vao;
  GLuint fs_vbo;
  GLuint fs_frag;
  GLuint fs_vert;
  GLuint fs_prog;
 
  GLuint fbo;
  GLuint depth;
  GLuint scene_tex;
  GLuint angle_tex;
 
  /* brush */
  GLuint br_prog;
  GLuint br_vert;
  GLuint br_frag;
  GLuint br_vao;
 
  int num_particles;
  GLuint tbo_position;
  GLuint tbo_tex;
  std::vector<GLfloat> positions;
};
 
#endif
#include <assert.h>
#include <roxlu/core/Log.h>
#include "Oils.h"
 
#define USE_ANGLES
 
Oils::Oils() 
  :win_w(0)
  ,win_h(0)
  ,br_prog(0)
  ,br_vert(0)
  ,br_frag(0)
  ,br_vao(0)
  ,num_particles(100000)
  ,tbo_position(0)
  ,tbo_tex(0)
  ,fs_vao(0)
  ,fs_vbo(0)
  ,fs_frag(0)
  ,fs_vert(0)
  ,fs_prog(0)
  ,fbo(0)
  ,depth(0)
  ,scene_tex(0)
  ,angle_tex(0)
{
}
 
Oils::~Oils() {
  RX_ERROR("Cleanup vao, prog, vert etc..");
}
 
bool Oils::setup(int winW, int winH) {
  glr_init();
 
  assert(winW);
  assert(winH);
  assert(br_prog == 0);
 
  win_w = winW;
  win_h = winH;
 
  br_vert = glCreateShader(GL_VERTEX_SHADER); glShaderSource(br_vert, 1, &OS_VS,NULL); glCompileShader(br_vert); eglGetShaderInfoLog(br_vert);
  br_frag = glCreateShader(GL_FRAGMENT_SHADER); glShaderSource(br_frag, 1, &OS_FS,NULL); glCompileShader(br_frag); eglGetShaderInfoLog(br_frag);
  br_prog = glCreateProgram(); glAttachShader(br_prog,br_vert); glAttachShader(br_prog,br_frag); glLinkProgram(br_prog); eglGetShaderLinkLog(br_prog);
  glUseProgram(br_prog);
 
  float pm[16] = { 0 };
  rx_ortho_top_left(win_w, win_h, -0.1f, 1.0f, pm);
 
  GLint u_pm = glGetUniformLocation(br_prog, "u_pm");
  if(u_pm < 0) {
    RX_ERROR("Cannot find u_pm");
    return false;
  }
  glUniformMatrix4fv(u_pm, 1, GL_FALSE, pm);
 
  // Pos sampler
  GLint u_pos_buffer = glGetUniformLocation(br_prog, "u_pos_buffer");
  if(u_pos_buffer < 0) {
    RX_ERROR("Cannot find u_pos_buffer");
    return false;
  }
  glUniform1i(u_pos_buffer, 0);
 
  // Angle tex
  GLint u_angle_tex = glGetUniformLocation(br_prog, "u_angle_tex");
  if(u_angle_tex < 0) {
    RX_ERROR("Canot find u_angle_tex");
    return false;
  }
  glUniform1i(u_angle_tex, 1);
 
  // Scene tex
  GLint u_scene_tex = glGetUniformLocation(br_prog, "u_scene_tex");
  if(u_scene_tex < 0) {
    RX_ERROR("Cannot find u_scene_tex");
    return false;
  }
  glUniform1i(u_scene_tex, 2);
 
  glGenVertexArrays(1, &br_vao);
 
 
  // TBO
  glGenTextures(1, &tbo_tex);
  glBindTexture(GL_TEXTURE_BUFFER, tbo_tex);
 
  glGenBuffers(1, &tbo_position);
  glBindBuffer(GL_TEXTURE_BUFFER, tbo_position);
  glBufferData(GL_TEXTURE_BUFFER, num_particles * sizeof(GLfloat) * 2, NULL, GL_DYNAMIC_DRAW);
 
  glTexBuffer(GL_TEXTURE_BUFFER, GL_RG32F, tbo_position);
  generateBrushVertices();
 
 
  // ANGLES
  // --------------------------------------------------------------
 
  // FBO
  glGenFramebuffers(1, &fbo);
  glBindFramebuffer(GL_FRAMEBUFFER, fbo);
 
  glGenRenderbuffers(1, &depth);
  glBindRenderbuffer(GL_RENDERBUFFER, depth);
  glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT32, win_w, win_h);
  glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_RENDERBUFFER, depth);
 
  glGenTextures(1, &scene_tex);
  glBindTexture(GL_TEXTURE_2D, scene_tex);
  glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, win_w, win_h, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
  glFramebufferTexture(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, scene_tex, 0);
 
  glGenTextures(1, &angle_tex);
  glBindTexture(GL_TEXTURE_2D, angle_tex);
  glTexImage2D(GL_TEXTURE_2D, 0, GL_R32F, win_w, win_h, 0, GL_RED, GL_FLOAT, NULL);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
  glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
  glFramebufferTexture(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT1, angle_tex, 0);
 
  RX_VERBOSE("OILS SCENE TEX: %d", scene_tex);
  RX_VERBOSE("OILS ANGLE TEX: %d", angle_tex);
 
  GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
  if(status != GL_FRAMEBUFFER_COMPLETE) {
    RX_ERROR("Framebuffer for Oil is not complete.");
    return false;
  }
 
  // FULLSCREEN BUFFER
  GLfloat buf[] = {
    -1.0, -1.0, 
    1.0, -1.0, 
    1.0, 1.0, 
    -1.0, -1.0,
    1.0, 1.0,
    -1.0, 1.0 
  };
 
  glGenVertexArrays(1, &fs_vao);
  glBindVertexArray(fs_vao);
 
  glGenBuffers(1, &fs_vbo);
  glBindBuffer(GL_ARRAY_BUFFER, fs_vbo);
  glBufferData(GL_ARRAY_BUFFER, sizeof(buf), buf, GL_STATIC_DRAW);
 
  glEnableVertexAttribArray(0); // pos
  glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, sizeof(GLfloat) * 2, (GLvoid*)0);
 
  glBindVertexArray(0);
 
  // FIND ANGLES PROG
  fs_vert = glCreateShader(GL_VERTEX_SHADER);
  glShaderSource(fs_vert, 1, &OSA_VS, NULL);
  glCompileShader(fs_vert); eglGetShaderInfoLog(fs_vert);
 
  fs_frag = glCreateShader(GL_FRAGMENT_SHADER);
  glShaderSource(fs_frag, 1, &OSA_FS, NULL);
  glCompileShader(fs_frag); eglGetShaderInfoLog(fs_frag);
 
  fs_prog = glCreateProgram();
  glAttachShader(fs_prog, fs_vert);
  glAttachShader(fs_prog, fs_frag);
  glBindAttribLocation(fs_prog, 0, "a_pos");
  glLinkProgram(fs_prog); eglGetShaderLinkLog(fs_prog);
  glUseProgram(fs_prog);
 
  GLint u_scene = glGetUniformLocation(fs_prog, "u_scene");
  if(u_scene < 0) {
    RX_ERROR("Cannot find u_scene");
    return false;
  }
  glUniform1i(u_scene, 0);
 
  return true;
}
 
void Oils::generateBrushVertices() {
  assert(win_w);
  assert(win_h);
 
 
  float max_w = win_w;
  float max_h = win_h;
  for(int i = 0; i < num_particles; ++i) {
    float x = rx_random(0.0f, max_w);
    float y = rx_random(0.0f, max_h);
    positions.push_back(x);
    positions.push_back(y);
   }
 
  glBindBuffer(GL_TEXTURE_BUFFER, tbo_position);
  glBufferSubData(GL_TEXTURE_BUFFER, 0, positions.size() * sizeof(GLfloat), &positions[0]);
 
}
void Oils::blitScene() {
  GLenum draw_bufs[] = { GL_COLOR_ATTACHMENT0 };
  glBindFramebuffer(GL_DRAW_FRAMEBUFFER, fbo);
  glDrawBuffers(1, draw_bufs);
  glViewport(0,0,win_w,win_h);
  glBlitFramebuffer(0, 0, win_w, win_h, 0, 0, win_w, win_h, GL_COLOR_BUFFER_BIT, GL_LINEAR);
 
  glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
}
 
void Oils::findAngles() {
  glViewport(0,0,win_w, win_h);
  GLenum draw_bufs[] = { GL_COLOR_ATTACHMENT1 };
  glBindFramebuffer(GL_DRAW_FRAMEBUFFER, fbo);
  glDrawBuffers(1, draw_bufs);
  glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
 
  glActiveTexture(GL_TEXTURE0);
  glBindTexture(GL_TEXTURE_2D, scene_tex);
 
  glBindVertexArray(fs_vao);
  glUseProgram(fs_prog);
  glDrawArrays(GL_TRIANGLES, 0, 6);
 
  glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
  glDrawBuffer(GL_BACK_LEFT);
}
 
void Oils::draw() {
 
  glActiveTexture(GL_TEXTURE0);
  glBindTexture(GL_TEXTURE_BUFFER, tbo_tex);
  glTexBuffer(GL_TEXTURE_BUFFER, GL_RG32F, tbo_position);
 
  glActiveTexture(GL_TEXTURE1);
  glBindTexture(GL_TEXTURE_2D, angle_tex);
 
  glActiveTexture(GL_TEXTURE2);
  glBindTexture(GL_TEXTURE_2D, scene_tex);
 
  glUseProgram(br_prog);
  glBindVertexArray(br_vao);
 
  glDrawArraysInstanced(GL_TRIANGLE_STRIP, 0, 4, num_particles);
 
}