Tuesday, May 4, 2010

Gesture capture and playback code

I implemented recording and musical level recognition on a basic level in SC. Code & screen shot of a small information GUI below:

(
var numinchannels, samplerate, controlrate, masterVol, waitTime, bufferArray, longbufferLength, muteInput, in, out, st_out, timeOffset, numOfPlayers, numOfTriggers, pathprefix, muteSched, playersSched, numPlayersToUnmute, pitchedAmtTrigger, treshLo, treshHi, inputBoost, oldstarttime, retriggerThreshold, minimalEventDuration, currentEventNumber = 0; //general variables
var recorder, play, straightPlayer, warpPlayer, rec1, time, playCnt, checkPlayers, checkDuration, addToList, phrasePlay, clear; //synthdefs and functions
var longbuf, fftbuf, fftbuflength; //buffers
var durationTriggerValue, dursMult, dursMultRand, stimes, stimesRand, timesEnv, timesRand, rates, ratesRand, pitches, pitchesRand, az, azRand, az2diff, el, elRand, el2diff, modAmts, modAmtsRand, modFreqs, modFreqsRand, atts, attsRand, rels, relsRand, amps, ampsRand, numPlayersToUnmuteEnv; //envelopes
var window1, midiNoteKnob, midiAvKnob, hasFreqAvKnob, ampAvKnob, loudnessAvKnob, amplitudeKnob, starttimeNumber, durationNumber, recButton, playButton, onsetNumber, playNumber, clearButton, phraseNumber; //gui
var startTime = 0, stopTime, midiNote, midiAv, hasFreqAv, ampAv, loudnessAv, amplitude, duration; //continous values
var thisPhrase, mainList; //arrays and lists with events
var num = 0;

// ----------- variables to set ------------------------- //
in = 0; //number of the input channel
out = 42; //number of the first channel of Ambisonics B-format output
st_out = 0; //number of the first channel of stereo out
masterVol = 0; //in dB
inputBoost = 0; //input amplification in dB
minimalEventDuration = 0.2; //in seconds
longbufferLength = 90; //length of rec buffers in seconds

numinchannels = 1; //currently only one input channel is supported
inputBoost = inputBoost.dbamp; //convert to amp
samplerate = Server.default.sampleRate; //get samplerate
controlrate = samplerate / Server.default.options.blockSize; //get control rate
if (samplerate < 50000, {fftbuflength = 1024}, {if (samplerate < 100000, {fftbuflength = 2048},  {fftbuflength = 4096})}); // set the FFT buffer size // fftbuflength.postln;
longbuf = CtkBuffer.buffer(size: samplerate * longbufferLength).load(sync: true); //main recording buffer
thisPhrase = Array.with(0, 0, 0, 0, 0, 0, 0, 0); //values: startTime, midiNote, midiAv, hasFreqAv, ampAv, loudnessAv, duration
mainList = Array.newClear(10000); //set uo the global event list for the buffer
timeOffset = thisThread.seconds; // get runtime time
time = thisThread.seconds - timeOffset; //get time relative to start

// SYNTHDEFS

recorder = CtkSynthDef(\recorder, {arg mute = 0, onsetTreshold = 0.2, treshHi = -46, treshLo = -52, averageDelay = 0.1; //tresholds to be adjusted
    var recTrigger, stopTrigger, amplitude, timer, hasFreq, freq, trigger, freqSlope, sig, hasFreqAv, ampAv, freqAv, counter, recTrigger0, inGate, chain, localbuf, onsets, midiNote, midiSlope, midiAv, loudness, loudnessAv;
    inGate = Gate.ar(SoundIn.ar(in) * inputBoost, mute.neg + 1) * (mute.neg + 1); //incoming sound with mute
    amplitude = Amplitude.kr(LagUD.ar(inGate, 0.01, 0.4), 0.01, 1.8); //amplitude tracking with "smoothing"
    trigger = Schmidt.kr(amplitude, treshLo.dbamp, treshHi.dbamp); //general trigger
    sig = inGate; //Gate.ar(inGate, trigger);
//    env = EnvGen.kr(Env.new([0, 1, 1, 0], [0.001, bufferslength - 0.002, 0.001]), (trigger * 2.001) - 1.001); //envelope for various purposes
//    recTrigger = trigger * env; //recTrigger not longer that buffer length
//    stopTrigger = recTrigger.neg +1; //for sending out messages
//    counter = PulseCount.kr(Delay1.kr(stopTrigger)) % 16; //counter to switch RecordBufs
//    counter = PulseCount.kr(stopTrigger) % 1;
    localbuf = LocalBuf.new(fftbuflength, 1); //fft buffer
    chain = FFT(localbuf, SoundIn.ar(in)); //start the chain
    onsets = Gate.kr(Onsets.kr(chain, onsetTreshold), trigger); //onset detection
    loudness = Loudness.kr(chain); //loudness analysis
//    stopTrigger = onsets.neg + 1;
    RecordBuf.ar(sig, longbuf, 0, run: 1, loop: 0, trigger: 1); //recorder
    #freq, hasFreq = Tartini.kr(sig); //pitch analysis
    midiNote = freq.cpsmidi; //convert to midi notes
    freqSlope = Slope.kr(freq).abs; //kind of amount of change of freq
    midiSlope = Slope.kr(midiNote).abs; //kind of amount of change of midi note
    midiAv = Gate.kr(AverageOutput.kr(midiSlope, DelayC.kr(onsets, 2, averageDelay)), trigger); //average of midi note change
    freqAv = Gate.kr(AverageOutput.kr(freqSlope, DelayC.kr(onsets, 2, averageDelay)), trigger); //average base freq in the sample
    hasFreqAv = Gate.kr(AverageOutput.kr(hasFreq, DelayC.kr(onsets, 2, averageDelay)), trigger); //average amount of pitched sounds in the sample
    ampAv = AverageOutput.kr(amplitude, DelayC.kr(onsets, 2, averageDelay)); //average amplitude of the sample
    loudnessAv = AverageOutput.kr(loudness, DelayC.kr(onsets, 2, averageDelay)); //average lodness
    timer = Phasor.kr(1, controlrate.reciprocal, 0, longbufferLength); //timer to report rec time
    SendReply.kr(onsets, 'start' , [timer]); //send start times for events
    SendReply.kr(Impulse.kr(5), 'cont' , [midiNote, midiAv, hasFreqAv, ampAv, loudnessAv, amplitude]); //send continous values for events
    SendReply.kr(trigger.neg + 1, 'stop' , [timer]); //send start times for events
    });

straightPlayer = CtkSynthDef(\straightPlayer, {arg dur, buffer = longbuf, startPos, amp = 0, rate = 1, az = 0, el = 0, dir=1, modAmt = 0, modFreq = 0, att = 0.001, rel = 0.001;
    var src, env, enc_w, enc_x, enc_y, enc_z, decoded, mod, secondLeg;
    secondLeg = dur - (dur * att) - (dur * rel);
    env = EnvGen.kr(Env([0.001, 1, 1, 0.001], [att * dur, secondLeg, rel * dur], \exponential));
    az = (az * pi) / 180; //to use az in degrees
    el = (el * pi) / 180; //to use el in degrees
    mod = SinOsc.ar(modFreq, modAmt);
//    src = PlayBuf.ar(numinchannels, buffer, (rate + mod) * BufRateScale.kr(buffer), 1, startPos * numinchannels * samplerate, 0) * env;
    src = PlayBuf.ar(numinchannels, buffer, rate * BufRateScale.kr(buffer), 1, startPos * samplerate, 0) * env;
//    src = PlayBuf.ar(1, buffer);
    # enc_w, enc_x, enc_y, enc_z = BFEncode1.ar(src, az, el, 1, 1, 0);
    Out.ar(st_out, B2UHJ.ar(enc_w, enc_x, enc_y, enc_z) * amp.dbamp * masterVol.dbamp); //stereo out
//    Out.ar(out, [enc_w, enc_x, enc_y, enc_z] * amp.dbamp * masterVol.dbamp); //ambisonics encoded out
    });

rec1 = recorder.new; //initialize recorder
rec1.play; //start recorder

//FUNCTIONS

addToList = {arg advance = 0;
    if(advance > 0, {//currentEventNumber.postln; thisPhrase.postln;
    mainList.put(currentEventNumber, thisPhrase.copy); //mainList.postln;
    currentEventNumber = currentEventNumber + 1;}, {mainList.put(currentEventNumber, thisPhrase)});
    };
 
checkDuration = {arg type;
    //type: 0 - stopTime, 1 = startTime
    if(type==0, {duration = stopTime - startTime; thisPhrase.put(6, duration); addToList.value(1); {durationNumber.value = duration; recButton.value = 0}.defer},
            {if(stopTime < oldstarttime, {duration = startTime - oldstarttime; thisPhrase.put(6, duration); addToList.value(1); {durationNumber.value = duration; recButton.value = 0}.defer})}
        );
    };
  
phrasePlay = {arg phraseNo;
    var playArray;
    'array no: '.post; phraseNo.postln;
    playArray = mainList[phraseNo];
    'play array: '.post; playArray.postln;
//    playArray[0].postln;
//    playArray[6].postln;
    straightPlayer.new(0, playArray[6]).dur_(playArray[6]).startPos_(playArray[0]).play;
    };

clear = { // free the buffer and the window
    longbuf.free;
    window1.close;
//    Server.default.quit;
//    Server.default.boot;
    };

//var startTime, stopTime, midiNote, midiAv, hasFreqAv, ampAv, loudnessAv, amplitude, duration;
// ------ a simple GUI --------
//starttimeNumber, durationNumber, recButton, playButton
    window1 = Window.new('Improviser parameters').front;
    window1.view.decorator=FlowLayout(window1.view.bounds);
    midiNoteKnob = EZKnob(window1, 50@90,"midiNote",    [0, 127]);
    midiAvKnob = EZKnob(window1, 50@90,"midiAv",    [0, 127]);
    hasFreqAvKnob = EZKnob(window1, 50@90,"hasFreq",    [0, 1]);
    ampAvKnob = EZKnob(window1, 50@90,"ampAv",    [0, 1]);
    loudnessAvKnob = EZKnob(window1, 50@90,"loudAv",    [0, 80]);
    amplitudeKnob = EZKnob(window1, 50@90," amp ",    [0, 1]);
    recButton = Button(window1, Rect(20,20,50,30)).states_([["waiting", Color.gray, Color.black], ["phrase", Color.black, Color.red]]);
    starttimeNumber = EZNumber(window1, 150@20,"s_time (s)", [0, longbufferLength]);
    durationNumber = EZNumber(window1, 150@20,"dur (s)", [0, longbufferLength]);
    onsetNumber = EZNumber(window1, 150@20,"onset no", [0, 10000]);
//    phraseNumber = EZNumber(window1, 150@20,"phrase no", [0, 10000]);
    playNumber = EZNumber(window1, 150@20,"play phrase no", [0, 10000], {|ez| phrasePlay.value(ez.value)}, labelWidth: 120);
    clearButton = Button(window1, Rect(20,20,50,30)).states_([["kill", Color.gray, Color.black], ["me", Color.black, Color.red]]).action_({ arg butt; clear.value});  

// RESPONDERS
OSCresponder(nil, 'start', { arg t, r, msg;// msg.postln;
        oldstarttime = startTime; //remember previous start time
        startTime = msg.at(3); //get actual start time
//        if(stopTime < oldstarttime, {duration = startTime - oldstarttime; {durationNumber.value = duration; recButton.value = 0}.defer; addToList.value(1)});
        checkDuration.value(1);     //check if we need to take duration from it
        thisPhrase.put(0, startTime); //put start time into the current phrase array
//        mainList.put(num, [startTime, 12345]);
//        'starttime: '.post; startTime.postln;
        {recButton.value = 1;
        starttimeNumber.value = startTime;
        onsetNumber.value = currentEventNumber;
        }.defer; //gui control
//        'main list: '.post;
//        mainList.postln;
//        num = num + 1;
    }).add;

OSCresponder(nil, 'stop', { arg t, r, msg;// msg.postln;
//    var stoptime;
        stopTime = msg.at(3); //get stop time
        checkDuration.value(0); //put duration in the array
//        'stoptime: '.post; stopTime.postln;
        {recButton.value = 0;
        durationNumber.value = duration;
        }.defer; //gui contol
    }).add;
  
OSCresponder(nil, 'cont', { arg t, r, msg;// msg.postln;
    var freqAv, hasFreqAv, ampAv, msgNew;
//    [midiNote, midiAv, hasFreqAv, ampAv, loudnessAv, amplitude]
        {
        midiNoteKnob.value = msg.at(3);
        midiAvKnob.value = msg.at(4);
        hasFreqAvKnob.value = msg.at(5);
        ampAvKnob.value = msg.at(6);
        loudnessAvKnob.value = msg.at(7);
        amplitudeKnob.value = msg.at(8);
        }.defer; //gui control
//        'org array: '.post; msg.postln;
        msgNew = msg.rotate(-3).clipExtend(5); //extract only useful values from the array
//        'new array: '.post; msgNew.postln;
        msgNew.do{arg val, i; thisPhrase.put(i+1, val.round(0.1))};//put parameters into the current phrase array
//        'this phrase: '.post; thisPhrase.postln;
//        addToList.value(0); //update current phrase in the main list
//        'current event no: '.post; currentEventNumber.postln;
    }).add;
)

No comments:

Post a Comment