Subscribe Bookmark
Craige_Hales

Staff

Joined:

Mar 21, 2013

FFT Video

This JSL makes a video to go with some music.  Usually it would be the other way around:  the music would be selected to match the video.  By selecting the music first, and limiting what the video can look like a bit, I get a set of images that can be combined with the audio (using Blender in this case) to get perfect synchronization.  Once again, the JSL is not cleaned up, but there are some comments sprinkled through it.


DoNotEat - YouTube

10856_closeUp.png

wav="\VBOXSVR\windowsShare\m\SpyGroove.wav"; // this is the music file

open(wav); // we'll get back to the music after a bit.  for now, start it playing.

// here's *some* of the parameters that were lifted out of the JSL as it was

// developed, moved up front to be easier to change

resolution = 120; // 20 makes a fast render but a less smooth appearance. 120 is slow and pretty

// this is the dimension of the mesh that creates the shape of the lumps.  doesn't seem to affect speed.

dim = 30; // believe 30 is a limitation on win opengl. needs to be even number.

// the 1920*1080 video size is still inline, but this made it possible to scale it to twice the size and

// let Blender scale it back down, probably smoothing out some jaggies...anti-aliasing...

picscale=2;//1;//

// initial rotation of the scene.  Later these numbers will be changed, a little, as the images are generated.

yrot=188; // +/- 20

xrot=68; // +/- 20

zrot=-310; // this can rotate continuously

// make some extra images at the beginning before the audio; this audio file has no leading silence

preroll=10;

// this was taken from the teapot example that ships with JMP, and modified a lot.  Some names stuck.

// in the teapot, a patch was a 4x4 matrix of numbers.  Here it is 30x30 (dim, above)

// this could be *way* simpler if you don't need it to be pretty...this is all about smoothness.

patch = Function( {dent}, // -1<=dent<=1

  xpoints = J( dim, dim, 0 );

  ypoints = J( dim, dim, 0 );

  zpoints = J( dim, dim, 0 );

  // build the smooth 3D shape

  // in the 30x30 square grid there are 15 concentric (square) rings to place in the desired sheet

  // for each point in the 30x30 grid, the ring number is determined by the distance to the

  // edge of the row or col closest to the edge

  bottom = 0; // starting here...trial and error to get a pleasing shape

  top = (dim / 2) - 1; // can't explain it because it is a week later...

  center = (dim + 1) / 2; // NOTE: the 30x30 x,y,z coordinates are

  scale = 5; // control points...they DO NOT LIE on the surface

  For( row = 1, row <= dim, row++,

  For( col = 1, col <= dim, col++,

  ring = Interpolate( Min( row - 1, col - 1, dim - row, dim - col ), bottom, 0.001, top, Pi() );

  ypoints[row, col] = scale * Sin( ring ) * (row - center) / dim;

  xpoints[row, col] = scale * Sin( ring ) * (col - center) / dim;

  zpoints[row, col] = Interpolate( ring,

  0, .9, // low values of ring are the folded-under corners of the

  .7, 1, // sheet.  .9 actually punches back up slightly from the base at 1.

  1.8, -.5, // mid values of ring from .7 to 1.8 map further negative

  // ring from 1.8 to PI runs from -.5 to either -2 (big bump for negative dent) or

  // +1.8 (big dent for positive dent) with 0 dent adjusted to -.3 to look flat

  Pi(), interpolate(-dent,-1,-2,0,-.3,1,1.8) // maxdented=1.8 -.3=centered bumped=-2

  );

  )

  );

  // build the combined points matrix.  shape() converts the 30x30 matrix into 900x1

  // and the 3 900x1 matrixes are joined into a 900x3

  points = Shape( ypoints, dim * dim, 1 ) || Shape( xpoints, dim * dim, 1 ) || Shape( zpoints, dim * dim, 1 );

// the MAP2 method adds the control points.  The WWW examples rarely explain the MAP2

// parameters in detail; I think I've got this mostly right, especially for a square case (xDim==yDim)

  Teapot << Map2(  MAP2_VERTEX_3,  0,  1,  N Cols( points ),  Sqrt( N Rows( points ) ),

           0,  1,  N Cols( points ) * Sqrt( N Rows( points ) ),  Sqrt( N Rows( points ) ),  points  );

// the EVALMESH2 method creates polygons with normal vectors using the resolution to

// determine the number of polygons

  Teapot << EvalMesh2( FILL, 0, resolution, 0, resolution );

);

// pre-build a set of objects.  Call the above function, a bunch of times, with a range of values from -1 to 1

// the objects are 3D display lists that can be reused.  No color is assigned yet.

nobj=240;// creates 2N+1 discrete levels.  turns out to be not important exactly how many.

obj = associativearray(); // to hold the display list objects

for(dent=-nobj, dent<=nobj, dent+=1,

  teapot=scenedisplaylist();

  patch(dent/nobj);

  obj[dent] = teapot;

);

// make the main 3D displaylist box.  To make the video not have a white border line I had to

// add 1.  Later you'll see where the border is removed

scene = Scene Box( picscale*1920+1, picscale*1080+1 );

New Window( "Teapot", vlistbox(scene, // the sliders were used to figure out what rotations I

sliderbox(-360,360,xrot,buildscene();,<<setwidth(1000)), // wanted; I never removed them

sliderbox(-360,360,yrot,buildscene();,<<setwidth(1000)), // and I'm careful not to adjust them

sliderbox(-360,360,zrot,buildscene();,<<setwidth(1000))) ); // during the picture generation

// this function gets called for each picture generated; it fills in the displaylists.

// I'm just rebuilding everything; it might be slightly better to only build the scene

// once and rebuild the teapot every time.

// I wish I'd renamed Teapot (the one way above) to Lump and (the one below) to Grid.

// buildscene makes a grid of lumps in the Teapot variable.

buildscene=function({},

scene << clear;

scene<<backgroundcolor(0,0,0);

// perspective and translate are changing the lens on the camera and

// moving it away from the origin (but still looking at the origin)

scene << perspective( 30, 1, 100 );

scene << translate(0,0,-30);

// set up four lights to shine on the grid of lumps

scene << Enable( Lighting );

scene << Enable( Light0 ); // this light is the main effect

scene << Light( Light0, AMBIENT, .01, .01, .01, 0 );

scene << Light( Light0, DIFFUSE, .6, .6, .6, 0 );

scene << Light( Light0, SPECULAR, .99, .99, .99, 0 );

scene << Enable( Light1 );

scene << Light( Light1, AMBIENT, .01, .01, .01, 0 );

scene << Light( Light1, DIFFUSE, .4, .4, .4, 0 );

scene << Light( Light1, SPECULAR, .1, .1, .1, 0 );

scene << Enable( Light2 );

scene << Light( Light2, AMBIENT, .01, .01, .01, 0 );

scene << Light( Light2, DIFFUSE, .1, .1, .1, 0 );

scene << Light( Light2, SPECULAR, .1, .1, .1, 0 );

scene << Enable( Light3 );

scene << Light( Light3, AMBIENT, .01, .01, .01, 0 );

scene << Light( Light3, DIFFUSE, .1, .1, .1, 0 );

scene << Light( Light3, SPECULAR, .1, .1, .1, 0 );

// position the lights

scene << Light( Light0, Position, -100, 100, 5, 0 );

scene << Light( Light1, Position, 100, 100, 50, 0 );

scene << Light( Light2, Position, -100, -100, 50, 0 );

scene << Light( Light3, Position, 100, -100, 50, 0 );

// start the grid of lumps in a separate display list so scene can call it with a rotation

Teapot = Scene Display List( 0 );

Teapot << Enable( MAP2_VERTEX_3 );

Teapot << Enable( Auto_Normal );

Teapot << MapGrid2( resolution, 0, 1, resolution, 0, 1 );

Teapot << Enable( COLOR_MATERIAL );

Teapot << Material( Front, Specular, .5, .5, .5, 1 );

Teapot << Material( Front, Shininess, 50 );

Teapot << Disable( POLYGON_SMOOTH );

grid=9;

ifreq=0;

nbins=N Rows( amplFiltered ) / 2;

// Way below is the FFT code that creates 735 bins of 30Hz each (22050 is the

// maximum frequency in a 44100 sample per second wav file...30*735...)

// this table describes which bins go in which lumps.  Most of the 49 lumps have

// a single bin and represent 30Hz.  at the low end of a piano keyboard, the keys

// are about 3Hz apart.  30 HZ toward the middle, 300 toward the top.  About the

// last 6 lumps are off the top end of the keyboard.

picker=[1 1, 2 2, 3 3, 4 4, 5 5, 6 6, 7 7, 8 8, 9 9, 10 10, 11 11, 12 12, 13 13, 14 14,

15 15, 16 16, 17 17, 18 18, 19 19, 20 20, 21 21, 22 22, 23 23, 24 24, 25 25, 26 26,

27 27, 28 28, 29 29, 30 30, 31 31, 32 32, 33 33, 34 34, 35 35, 36 36,

37 38, 39 40, 41 42, 43 44, 45 46, 47 48, 49 50, 50 54, 55 60,

61 70, 71 99, 100 200, 201 735];

// toward the end of the development cycle (end of the weekend) I decided to rearrange

// the lumps into a spiral with low notes in the center...

freqs = [ // lay out the spiral path

  43 44 45 46 47 48 49

  42 21 22 23 24 25 26

  41 20  7  7  9 10 27

  40 19  6  1  2 11 28

  39 18  5  4  3 12 29

  38 17 16 15 14 13 30

  37 36 35 34 33 32 31

];

if(nbins!=picker[nrows(picker),2],throw("bad")); // hardcoded size of picker

// now the 49 lumps are generated in a 7x7 pattern

For( xx = -grid, xx <= grid, xx += 3,

  For( yy = -grid, yy <= grid, yy += 3,

  ifreq++; // index into the spiral matrix

  mm=picker[freqs[ifreq],0]; // mm is a [low high] pair of bin numbers from the FFT

// ampFiltered is an array of 735 bins of frequency info from the FFT

// this is a hand-crafted piece of art that determines what the lump's response is

  q=floor(max(-nobj,min(nobj,(log(max( amplFiltered[mm] ))-13)*60)));

  teapot << pushMatrix; // each lump saves the old transform matrix

  teapot << translate( xx, yy, 0 ); // each lump translates to its position

// each lump has a color for its frequency

  {red, green, blue} = Color To RGB( HLS Color( Interpolate( freqs[ifreq], 1, 0, 49, 1 ), .5, 1 ) );

  Teapot << Color( red, green, blue );

  Teapot << callList( obj );// include another lump into the grid

  Teapot << popmatrix; // restore the transform

  )

);

scene<<rotate(yrot+9*sin(zrot/79),0,1,0); // xrot and yrot are small effects

scene<<rotate(xrot+9*sin(zrot/61),1,0,0); // compared to zrot

zrot+=.25;// degrees ... this is the source of the rotation on screen

scene<<rotate(zrot,0,0,1);

scene << ArcBall( Teapot, 10 ); // arcball allows hand adjustment

scene << update;scene<<inval;scene<<updatewindow; // force the update

);

//////////////////////

// as promised, back to the wav file.  get it into a matrix of numeric sample values

x = Load Text File( wav, blob );

x = Hex( x ); // doubles the size of the data, but the hex characters are easy to parse

If( Pat Match( x, Hex( "RIFF" ) + Pat Arb() + Hex( "data" ), "" ), // found the data chunk

  x = Blob To Matrix( Hex To Blob( x ), "int", 2, "little", nCols = 2 );

  x = (x[0, 1] + x[0, 2]) / 2; // mono

// not cols...these are bins.  this is 1470, but only the first half are used (735)

  cols = (44100 / 30); // number of samples in 1/30 second (video will be 30 frames/sec)

  rows = N Rows( x ); // this is the number of samples in the wav

  rows = Floor( rows / cols ); // number of frames in video

  x = Shape( x, rows, cols ); // each row is a frame and has all the samples that belong to the frame

  period = cols / 44100; // ~ .0333 second

  binwidth = 1 / period; // 30 Hz per bin

  maxs = J( 15, 1, 0 ); // unused now?

  maxa = 0; // collect the maximum amplitude for scaling

  For( i = 1, i <= rows, i++,

  {real, imag} = FFT( {x[i, 0]`} );

  ampl = Sqrt( real :* real + imag :* imag ); // just the amplitude

  time = (i - 1) * (cols / 44100); // unused?

  maxa = Max( maxa, Max( ampl ) );

  );

  // do it again, this time make the pics using the maxs and maxa

i=1;

zrot= zrot-preroll*.25;

{real, imag} = FFT( {x[i, 0]`} );

  ampl = Sqrt( real :* real + imag :* imag ); // just the amplitude

  amplFiltered = ampl*0.00001;

// make preroll lead-in frames before the audio;

for(i=-preroll,i<1,i++,

buildscene();wait(0);

pic=scene<<getpicture;

pixels = pic<<getpixels;

pixels=pixels[2::picscale*1080+1,3::picscale*1920+2];

pic<<setpixels(pixels);

pic<<saveimage("\VBOXSVR\windowsShare\m\m2\m"||right(char(i+preroll,9),9,"0")||".png");

  amplFiltered = ampl *interpolate(i,-preroll,0.00001,-preroll/5,0.0001,1,1);

);

// make the frames from the filtered FFT results.  the filter allows fast rise but slow decay

  For( i = 1, i <= rows, i++,print(i/rows); // each for is a video frame and 1/30 second of audio

  {real, imag} = FFT( {x[i, 0]`} ); // FFT the row

  ampl = Sqrt( real :* real + imag :* imag ); // just the amplitude

  time = (i - 1) * (cols / 44100); // unused

  amplFiltered = if(i==1,ampl,

  selector=greater(ampl,amplFiltered);

  selector:*ampl + (1-selector):*((3*amplFiltered+ampl)/4));

buildscene();wait(0);

pic=scene<<getpicture;

pixels = pic<<getpixels;

// here's the clipping of the unwanted border pixels...

// the exact offsets may need to be rediscovered sometime if JMP or OS changes something

pixels=pixels[2::picscale*1080+1,3::picscale*1920+2];

pic<<setpixels(pixels);

// Blender works well with a directory of files x00001.png, x00002.png, ...

pic<<saveimage("\VBOXSVR\windowsShare\m\m2\m"||right(char(i+preroll,9),9,"0")||".png");

  );

// dead frames at end ramp down the amplitude...

  for(i,i<rows+360,i++, // trailer

  amplFiltered=amplFiltered:*.95;

buildscene();wait(0);

pic=scene<<getpicture;

pixels = pic<<getpixels;

pixels=pixels[2::picscale*1080+1,3::picscale*1920+2];

pic<<setpixels(pixels);

pic<<saveimage("\VBOXSVR\windowsShare\m\m2\m"||right(char(i+preroll,9),9,"0")||".png");

  );

,

  Print( "bad file" )

);

print("done");

Article Tags